feat: log failed projects

README.md

@@ -11,7 +11,7 @@ Snyk API project importer
 
 
 ### Setup
-`npm i && npm run build`
+`npm i`
 
 ### Running the test locally
 You will need to set the following environment variable:
@@ -21,4 +21,10 @@ You will need to set the following environment variable:
 Run the tests with `npm test`
 
 
+### Running the script
+- `SNYK_HOST`
+- `SNYK_API_TOKEN` - your [Snyk api token](https://app.snyk.io/account)
+- `SNYK_LOG_PATH` - the path to folder where all logs should be saved
+- `CONCURRENT_IMPORTS` (optional) defaults to 5 which is the recommended amount to import at once as a max.
+
 

src/common.ts

@@ -1,3 +1,4 @@
 export const IMPORT_LOG_NAME = 'imported-targets.log';
 export const IMPORT_PROJECTS_FILE_NAME= 'import-projects.json';
 export const FAILED_LOG_NAME = 'failed-imports.log';
+export const FAILED_PROJECTS_LOG_NAME = 'failed-projects.log';

src/lib/import/index.ts

@@ -19,7 +19,12 @@ export async function importTarget(
   target: Target,
   files?: FilePath[] | undefined,
   loggingPath?: string,
-): Promise<{ pollingUrl: string }> {
+): Promise<{
+  pollingUrl: string;
+  target: Target;
+  orgId: string;
+  integrationId: string;
+}> {
   const apiToken = getApiToken();
   debug('Importing:', JSON.stringify({ orgId, integrationId, target }));
 
@@ -63,7 +68,12 @@ export async function importTarget(
     }
     debug(`Received locationUrl for ${target.name}: ${locationUrl}`);
     logImportedTarget(orgId, integrationId, target, locationUrl, loggingPath);
-    return { pollingUrl: locationUrl };
+    return {
+      pollingUrl: locationUrl,
+      integrationId,
+      target,
+      orgId,
+    };
   } catch (error) {
     logFailedImports(orgId, integrationId, target, loggingPath);
     const err: {
@@ -81,7 +91,6 @@ export async function importTargets(
   loggingPath = getLoggingPath(),
 ): Promise<string[]> {
   const pollingUrls: string[] = [];
-  // TODO: filter out previously processed
   // TODO: validate targets
   await pMap(
     targets,
@@ -95,7 +104,6 @@ export async function importTargets(
           files,
           loggingPath,
         );
-        // TODO: log all succeeded into a file
         pollingUrls.push(pollingUrl);
       } catch (error) {
         // TODO: log all failed into a file

src/lib/poll-import/index.ts

@@ -6,6 +6,7 @@ import * as _ from 'lodash';
 import * as pMap from 'p-map';
 import { PollImportResponse, Project } from '../types';
 import { getApiToken } from '../get-api-token';
+import { logFailedProjects } from '../../log-failed-projects';
 
 const debug = debugLib('snyk:poll-import');
 const MIN_RETRY_WAIT_TIME = 3000;
@@ -81,10 +82,17 @@ export async function pollImportUrls(
     uniqueLocationUrls,
     async (locationUrl) => {
       try {
-        const projects = await pollImportUrl(locationUrl);
+        const allProjects = await pollImportUrl(locationUrl);
+        const [failedProjects, projects] = _.partition(
+          allProjects,
+          (p: Project) => !p.success,
+        );
+        logFailedProjects(locationUrl, failedProjects);
         // TODO: log all succeeded into a file
         projectsArray.push(...projects);
       } catch (error) {
+        // logFailedProjects(locationUrl, failedProjects);
+
         // TODO: log all failed into a file
         debug('Failed to poll:', locationUrl);
       }

src/log-failed-projects.ts

@@ -0,0 +1,20 @@
+import * as fs from 'fs';
+import * as _ from 'lodash';
+import { FAILED_PROJECTS_LOG_NAME } from './common';
+import { Project } from './lib/types';
+import { getLoggingPath } from './lib/get-logging-path';
+
+export async function logFailedProjects(
+  locationUrl: string,
+  projects: Project[],
+  loggingPath: string = getLoggingPath(),
+): Promise<void> {
+  try {
+    projects.forEach((project) => {
+      const log = `${locationUrl}:${Object.values(_.omit(project)).join(':')},`;
+      fs.appendFileSync(`${loggingPath}/${FAILED_PROJECTS_LOG_NAME}`, log);
+    });
+  } catch (e) {
+    // do nothing
+  }
+}

src/scripts/import-projects.ts

@@ -65,8 +65,8 @@ export async function ImportProjects(
       targetIndex,
       targetIndex + concurrentTargets,
     );
-    const pollingUrls = await importTargets(batch, loggingPath);
-    projects.push(...(await pollImportUrls(pollingUrls)));
+    const pollingUrlsAndContext = await importTargets(batch, loggingPath);
+    projects.push(...(await pollImportUrls(pollingUrlsAndContext)));
   }
 
   return projects;

test/scripts/fixtures/projects-with-errors/import-projects.json

@@ -0,0 +1,13 @@
+{
+  "targets": [
+    {
+      "orgId": "f0125d9b-271a-4b50-ad23-80e12575a1bf",
+      "integrationId": "c4de291b-e083-4c43-a72c-113463e0d268",
+      "target": {
+        "name": "projects-with-errors",
+        "owner": "snyk-fixtures",
+        "branch": "master"
+      }
+    }
+  ]
+}

test/scripts/import-projects.test.ts

@@ -6,6 +6,7 @@ import {
   IMPORT_PROJECTS_FILE_NAME,
   IMPORT_LOG_NAME,
   FAILED_LOG_NAME,
+  FAILED_PROJECTS_LOG_NAME,
 } from '../../src/common';
 
 describe('Import projects script', () => {
@@ -53,21 +54,29 @@ describe('Skips & logs issues', () => {
   const OLD_ENV = process.env;
 
   afterEach(() => {
-    const logPath = path.resolve(
+    const importsInitiatedLog = path.resolve(
       process.env.SNYK_LOG_PATH as string,
       IMPORT_LOG_NAME,
     );
-    const failedLogPath = path.resolve(
+    const importsFailedLog = path.resolve(
       process.env.SNYK_LOG_PATH as string,
       FAILED_LOG_NAME,
     );
+    const projectsFailedLog = path.resolve(
+      process.env.SNYK_LOG_PATH as string,
+      FAILED_PROJECTS_LOG_NAME,
+    );
     jest.clearAllMocks();
-    try {
-      fs.unlinkSync(failedLogPath);
-      fs.unlinkSync(logPath);
-    } catch (e) {
-      // do nothing
-    }
+    [importsFailedLog, importsInitiatedLog, projectsFailedLog].forEach(
+      (path) => {
+        try {
+          fs.unlinkSync(path);
+        } catch (e) {
+          // do nothing
+        }
+      },
+    );
+
     process.env = { ...OLD_ENV };
   });
 
@@ -83,7 +92,10 @@ describe('Skips & logs issues', () => {
       FAILED_LOG_NAME,
     );
     const projects = await ImportProjects(
-      path.resolve(__dirname + '/fixtures/invalid-target/import-projects-invalid-target.json'),
+      path.resolve(
+        __dirname +
+          '/fixtures/invalid-target/import-projects-invalid-target.json',
+      ),
     );
     expect(projects.length === 0).toBeTruthy();
     let logFile = null;
@@ -103,7 +115,10 @@ describe('Skips & logs issues', () => {
       process.env.SNYK_LOG_PATH as string,
       IMPORT_LOG_NAME,
     );
-    const failedLogPath = path.resolve(process.env.SNYK_LOG_PATH as string, FAILED_LOG_NAME);
+    const failedLogPath = path.resolve(
+      process.env.SNYK_LOG_PATH as string,
+      FAILED_LOG_NAME,
+    );
     process.env.SNYK_HOST = 'https://do-not-exist.com';
     // TODO: ensure all failures are logged & assert it is present in the log
     const projects = await ImportProjects(
@@ -121,6 +136,42 @@ describe('Skips & logs issues', () => {
     const failedLog = fs.readFileSync(failedLogPath, 'utf8');
     expect(failedLog).toMatch('ruby-with-versions');
   }, 300);
+  it('Logs failed projects', async () => {
+    const logRoot = __dirname + '/fixtures/projects-with-errors/';
+    process.env.SNYK_LOG_PATH = logRoot;
+    const logPath = path.resolve(
+      process.env.SNYK_LOG_PATH as string,
+      IMPORT_LOG_NAME,
+    );
+    const failedImportLogPath = path.resolve(
+      process.env.SNYK_LOG_PATH as string,
+      FAILED_LOG_NAME,
+    );
+    const failedProjectsLogPath = path.resolve(
+      process.env.SNYK_LOG_PATH as string,
+      FAILED_PROJECTS_LOG_NAME,
+    );
+    const projects = await ImportProjects(
+      path.resolve(
+        __dirname + '/fixtures/projects-with-errors/import-projects.json',
+      ),
+    );
+    const logFile = fs.readFileSync(logPath, 'utf8');
+    expect(logFile).not.toBeNull();
+    const failedProjectsLog = fs.readFileSync(failedProjectsLogPath, 'utf-8');
+    expect(failedProjectsLog).not.toBeNull();
+    expect(failedProjectsLog).toMatch(
+      'ruby-app-cyclic-lockfile-master/Gemfile.lock',
+    );
+
+    let failedImportLog = null;
+    try {
+      failedImportLog = fs.readFileSync(failedImportLogPath, 'utf8');
+    } catch (e) {
+      expect(failedImportLog).toBeNull();
+    }
+    expect(projects.length >= 1).toBeTruthy();
+  }, 50000);
 });
 
 describe('Error handling', () => {