INT-9750: change permission errors to warnings

src/config.ts

@@ -1,22 +1,42 @@
-import { IntegrationExecutionContext } from '@jupiterone/integration-sdk-core';
+import {
+  IntegrationExecutionContext,
+  IntegrationLogger,
+} from '@jupiterone/integration-sdk-core';
 import { ServiceUsageClient } from './steps/service-usage/client';
 import { IntegrationConfig, SerializedIntegrationConfig } from './types';
 import { deserializeIntegrationConfig } from './utils/integrationConfig';
+import { google } from 'googleapis';
+import { handleApiClientError } from './google-cloud/client';
+import { ServiceUsageListFilter } from './google-cloud/types';
 
-export async function executeTestRequest(config: IntegrationConfig) {
-  const googleClient = new ServiceUsageClient({ config });
+export async function executeTestRequest(
+  config: IntegrationConfig,
+  logger: IntegrationLogger,
+) {
+  try {
+    const client = google.serviceusage({ version: 'v1', retry: false });
+    const googleClient = new ServiceUsageClient({ config }, logger);
+    const auth = await googleClient.getAuthenticatedServiceClient();
 
-  return googleClient.collectEnabledServices();
+    await client.services.list({
+      parent: `projects/${config.serviceAccountKeyConfig.project_id}`,
+      pageSize: 200,
+      auth: auth,
+      filter: ServiceUsageListFilter.ENABLED,
+    });
+  } catch (err) {
+    throw handleApiClientError(err);
+  }
 }
 
 export async function validateInvocation(
   context: IntegrationExecutionContext<SerializedIntegrationConfig>,
 ) {
-  const { instance } = context;
+  const { instance, logger } = context;
   const { config: serializedIntegrationConfig } = instance;
   const config = (context.instance.config = deserializeIntegrationConfig(
     serializedIntegrationConfig,
   ));
 
-  await executeTestRequest(config);
+  await executeTestRequest(config, logger);
 }

src/getStepStartStates.ts

@@ -506,7 +506,10 @@ async function getStepStartStatesUsingServiceEnablements(params: {
   let enabledServiceNames: string[];
   let serviceAccountProjectEnabledServiceNames: string[];
   try {
-    const enabledServiceData = await enablement.getEnabledServiceNames(config);
+    const enabledServiceData = await enablement.getEnabledServiceNames(
+      config,
+      logger,
+    );
     enabledServiceNames = enabledServiceData.intersectedEnabledServices ?? [];
     serviceAccountProjectEnabledServiceNames =
       enabledServiceData.mainProjectEnabledServices ?? [];

src/google-cloud/client.test.ts

@@ -1,4 +1,5 @@
 import {
+  IntegrationLogger,
   IntegrationProviderAPIError,
   IntegrationProviderAuthorizationError,
 } from '@jupiterone/integration-sdk-core';
@@ -13,6 +14,7 @@ import {
 } from '../../test/recording';
 import { parseServiceAccountKeyFile } from '../utils/parseServiceAccountKeyFile';
 import { Client } from './client';
+import { getMockLogger } from '../../test/helpers/getMockLogger';
 
 describe('#getAuthenticatedServiceClient', () => {
   let googleAuthSpy: jest.SpyInstance<
@@ -53,7 +55,9 @@ describe('#getAuthenticatedServiceClient', () => {
 
     googleAuthSpy.mockReturnValueOnce(mockGoogleAuthClient);
 
-    const client = new Client({ config: instanceConfig });
+    const logger = getMockLogger<IntegrationLogger>();
+
+    const client = new Client({ config: instanceConfig }, logger);
 
     const auth = await client.getAuthenticatedServiceClient();
     const auth2 = await client.getAuthenticatedServiceClient();
@@ -94,9 +98,11 @@ describe('withErrorHandling', () => {
   let client;
   let onRetry;
 
+  const logger = getMockLogger<IntegrationLogger>();
+
   beforeEach(() => {
     onRetry = jest.fn();
-    client = new Client({ config, onRetry: onRetry });
+    client = new Client({ config, onRetry: onRetry }, logger);
   });
 
   [IntegrationProviderAuthorizationError, IntegrationProviderAPIError].forEach(
@@ -181,14 +187,16 @@ describe('withErrorHandling', () => {
 });
 
 describe('Client', () => {
+  const logger = getMockLogger<IntegrationLogger>();
+
   test('should set projectId to the config projectId if provided', () => {
     const configProjectId = 'projectId';
     const serviceAccountProjectId = 'serviceAccountProjectId';
     const config = {
       projectId: configProjectId,
       serviceAccountKeyConfig: { project_id: serviceAccountProjectId },
     } as unknown as IntegrationConfig;
-    expect(new Client({ config }).projectId).toBe(configProjectId);
+    expect(new Client({ config }, logger).projectId).toBe(configProjectId);
   });
 
   test('should set projectId to the service account projectId if the configprojectId is not provided', () => {
@@ -198,7 +206,9 @@ describe('Client', () => {
       projectId: configProjectId,
       serviceAccountKeyConfig: { project_id: serviceAccountProjectId },
     } as unknown as IntegrationConfig;
-    expect(new Client({ config }).projectId).toBe(serviceAccountProjectId);
+    expect(new Client({ config }, logger).projectId).toBe(
+      serviceAccountProjectId,
+    );
   });
 
   test('should set projectId to the projectId override option reguardless on if the service account projectId and config projectIds are provided or not', () => {
@@ -209,9 +219,9 @@ describe('Client', () => {
       projectId: configProjectId,
       serviceAccountKeyConfig: { project_id: serviceAccountProjectId },
     } as unknown as IntegrationConfig;
-    expect(new Client({ config, projectId: overrideProjectId }).projectId).toBe(
-      overrideProjectId,
-    );
+    expect(
+      new Client({ config, projectId: overrideProjectId }, logger).projectId,
+    ).toBe(overrideProjectId);
   });
 });
 

src/google-cloud/client.ts

@@ -1,7 +1,9 @@
 import {
   IntegrationError,
+  IntegrationLogger,
   IntegrationProviderAPIError,
   IntegrationProviderAuthorizationError,
+  IntegrationWarnEventName,
 } from '@jupiterone/integration-sdk-core';
 import { retry } from '@lifeomic/attempt';
 import { GaxiosError, GaxiosResponse } from 'gaxios';
@@ -44,12 +46,16 @@ export class Client {
   readonly projectId: string;
   readonly organizationId?: string;
   readonly folderId?: string;
+  readonly logger: IntegrationLogger;
 
   private credentials: CredentialBody;
   private auth: BaseExternalAccountClient;
   private readonly onRetry?: (err: any) => void;
 
-  constructor({ config, projectId, organizationId, onRetry }: ClientOptions) {
+  constructor(
+    { config, projectId, organizationId, onRetry }: ClientOptions,
+    logger: IntegrationLogger,
+  ) {
     this.projectId =
       projectId ||
       config.projectId ||
@@ -61,6 +67,7 @@ export class Client {
     };
     this.folderId = config.folderId;
     this.onRetry = onRetry;
+    this.logger = logger;
   }
 
   private async getClient(): Promise<BaseExternalAccountClient> {
@@ -91,19 +98,39 @@ export class Client {
     callback: (data: T) => Promise<void>,
   ) {
     return this.forEachPage(async (nextPageToken) => {
-      const result = await this.withErrorHandling(() => fn(nextPageToken));
-      await callback(result.data);
+      try {
+        const result = await this.withErrorHandling(() => fn(nextPageToken));
+        await callback(result.data);
 
-      return result;
+        return result;
+      } catch (err) {
+        if (err.status === 403) {
+          this.logger.warn(
+            { err },
+            `Step failed due to missing permission. Requires additional permission`,
+          );
+
+          this.logger.publishWarnEvent({
+            name: IntegrationWarnEventName.MissingPermission,
+            description: `Received authorization error when attempting to call ${err.endpoint}. Please review permissions in the integration documentation.`,
+          });
+          return;
+        }
+
+        throw err;
+      }
     });
   }
 
   async forEachPage<T>(
-    cb: (nextToken: string | undefined) => Promise<PageableGaxiosResponse<T>>,
+    cb: (
+      nextToken: string | undefined,
+    ) => Promise<PageableGaxiosResponse<T> | undefined>,
   ): Promise<any> {
     let nextToken: string | undefined;
     do {
       const response = await cb(nextToken);
+      if (!response) return;
       nextToken = response.data.nextPageToken
         ? response.data.nextPageToken
         : undefined;
@@ -151,7 +178,7 @@ export class Client {
 /**
  * Codes unknown error into JupiterOne errors
  */
-function handleApiClientError(error: GaxiosError) {
+export function handleApiClientError(error: GaxiosError) {
   // If the error was already handled, forward it on
   if (error instanceof IntegrationError) {
     return error;

src/steps/access-context-manager/index.ts

@@ -62,8 +62,9 @@ export async function fetchAccessPolicies(
   const {
     jobState,
     instance: { config },
+    logger,
   } = context;
-  const client = new AccessContextManagerClient({ config });
+  const client = new AccessContextManagerClient({ config }, logger);
   await client.iterateAccessPolicies(async (accessPolicy) => {
     await jobState.addEntity(createAccessPolicyEntity(accessPolicy));
   });
@@ -75,8 +76,9 @@ export async function fetchAccessLevels(
   const {
     jobState,
     instance: { config },
+    logger,
   } = context;
-  const client = new AccessContextManagerClient({ config });
+  const client = new AccessContextManagerClient({ config }, logger);
 
   await jobState.iterateEntities(
     {
@@ -333,8 +335,9 @@ export async function fetchServicePerimeters(
   const {
     jobState,
     instance: { config },
+    logger,
   } = context;
-  const client = new AccessContextManagerClient({ config });
+  const client = new AccessContextManagerClient({ config }, logger);
 
   await jobState.iterateEntities(
     {

src/steps/api-gateway/index.ts

@@ -53,9 +53,10 @@ export async function fetchApiGatewayApis(
   const {
     jobState,
     instance: { config },
+    logger,
   } = context;
 
-  const client = new ApiGatewayClient({ config });
+  const client = new ApiGatewayClient({ config }, logger);
 
   await client.iterateApis(async (api) => {
     const apiId = api.name?.split('/')[5];
@@ -78,7 +79,7 @@ export async function fetchApiGatewayApiConfigs(
     logger,
   } = context;
 
-  const client = new ApiGatewayClient({ config });
+  const client = new ApiGatewayClient({ config }, logger);
 
   await jobState.iterateEntities(
     {
@@ -151,9 +152,10 @@ export async function fetchApiGatewayGateways(
   const {
     jobState,
     instance: { config },
+    logger,
   } = context;
 
-  const client = new ApiGatewayClient({ config });
+  const client = new ApiGatewayClient({ config }, logger);
 
   await client.iterateGateways(async (gateway) => {
     const gatewayId = gateway.name?.split('/')[5];

src/steps/app-engine/index.ts

@@ -92,7 +92,7 @@ export async function fetchAppEngineApplication(
     instance: { config },
     logger,
   } = context;
-  const client = new AppEngineClient({ config });
+  const client = new AppEngineClient({ config }, logger);
   const { projectId } = client;
 
   let application: appengine_v1.Schema$Application | undefined;
@@ -223,7 +223,7 @@ export async function fetchAppEngineServices(
     return;
   }
 
-  const client = new AppEngineClient({ config });
+  const client = new AppEngineClient({ config }, logger);
   const { projectId } = client;
 
   await withAppEngineErrorHandling(logger, projectId, async () => {
@@ -251,7 +251,7 @@ export async function fetchAppEngineServiceVersions(
     logger,
   } = context;
 
-  const client = new AppEngineClient({ config });
+  const client = new AppEngineClient({ config }, logger);
   const { projectId } = client;
 
   await jobState.iterateEntities(
@@ -342,7 +342,7 @@ export async function fetchAppEngineVersionInstances(
     logger,
   } = context;
 
-  const client = new AppEngineClient({ config });
+  const client = new AppEngineClient({ config }, logger);
   const { projectId } = client;
 
   await jobState.iterateEntities(

src/steps/big-query/index.ts

@@ -62,8 +62,9 @@ export async function fetchBigQueryDatasets(
   const {
     jobState,
     instance: { config },
+    logger,
   } = context;
-  const client = new BigQueryClient({ config });
+  const client = new BigQueryClient({ config }, logger);
 
   try {
     await client.iterateBigQueryDatasets(async (dataset) => {
@@ -140,7 +141,7 @@ export async function fetchBigQueryModels(
     instance: { config },
     logger,
   } = context;
-  const client = new BigQueryClient({ config });
+  const client = new BigQueryClient({ config }, logger);
 
   await jobState.iterateEntities(
     {
@@ -183,7 +184,7 @@ export async function fetchBigQueryTables(
     logger,
     instance: { config },
   } = context;
-  const client = new BigQueryClient({ config });
+  const client = new BigQueryClient({ config }, logger);
 
   await jobState.iterateEntities(
     {