API Endpoint to Download TSV Files for donors by Donor ID

src/clinical/api/clinical-api.ts

@@ -36,7 +36,10 @@ import {
   ClinicalErrorsApiBody,
   ClinicalSearchApiBody,
   CompletionState,
+  DonorDataApiBody,
 } from './types';
+import { loggerFor } from '../../logger';
+const L = loggerFor(__filename);
 
 // TODO: Update value type to match mongo schema
 export const completionFilters: Record<CompletionState, any> = {
@@ -140,6 +143,40 @@ class ClinicalController {
     res.send(zip.toBuffer());
   }
 
+  /**
+   * Download Clinical Data selected by Donor ID.
+   * Requires Full Read Access.
+   * Used by API for a user to download clinical data for specific donors and/or files.
+   * @param req
+   * @param res
+   * @returns
+   */
+  @HasFullReadAccess()
+  async getDonorDataByIdAsTsvsInZip(req: Request, res: Response) {
+    const bodyParseResult = DonorDataApiBody.safeParse(req.body);
+    if (!bodyParseResult.success) {
+      return ControllerUtils.badRequest(
+        res,
+        `Invalid data in request body: ${JSON.stringify(bodyParseResult.error)}`,
+      );
+    }
+    const { donorIds } = bodyParseResult.data;
+
+    const donorEntityData = await service.getDonorEntityData(donorIds);
+
+    const date = currentDateFormatted();
+    const fileName = `filename=Donor_Clinical_Data_${date}.zip`;
+    res
+      .status(200)
+      .contentType('application/zip')
+      .attachment(fileName)
+      .setHeader('content-disposition', fileName);
+
+    const zip = createClinicalZipFile(donorEntityData);
+
+    res.send(zip.toBuffer());
+  }
+
   @HasProgramReadAccess((req: Request) => req.params.programId)
   async getProgramClinicalEntityData(req: Request, res: Response) {
     const programShortName: string = req.params.programId;

src/clinical/api/types.ts

@@ -34,3 +34,11 @@ export const ClinicalSearchApiBody = zod.object({
   submitterDonorIds: zod.array(zod.string().nonempty()).default([]),
 });
 export type ClinicalSearchApiBody = zod.infer<typeof ClinicalSearchApiBody>;
+
+/**
+ * Download Donor Data by ID
+ */
+export const DonorDataApiBody = zod.object({
+  donorIds: zod.array(zod.number().positive()).min(1),
+});
+export type DonorDataApiBody = zod.infer<typeof ClinicalDataApiBody>;

src/clinical/clinical-service.ts

@@ -28,6 +28,7 @@ import {
   ClinicalErrorsResponseRecord,
   EntityAlias,
   aliasEntityNames,
+  allEntityNames,
 } from '../common-model/entities';
 import { Errors } from '../utils';
 import { patchCoreCompletionWithOverride } from '../submission/submission-to-clinical/stat-calculator';
@@ -53,13 +54,14 @@ export type ClinicalDonorEntityQuery = {
   completionState?: {};
 };
 
-// Returns paginated Donor + Entity Submission Data
-export type PaginatedClinicalQuery = ClinicalDonorEntityQuery & {
+export type PaginationQuery = {
   page: number;
   pageSize?: number;
   sort: string;
 };
 
+export type PaginatedClinicalQuery = ClinicalDonorEntityQuery & PaginationQuery;
+
 // GQL Query Arguments
 // Submitted Data Search Bar
 export type ClinicalSearchVariables = {
@@ -122,6 +124,10 @@ export async function getDonors(programId: string) {
   return await donorDao.findBy({}, 999);
 }
 
+export async function getDonorsByIds(donorIds: number[]) {
+  return await donorDao.findByDonorIds(donorIds);
+}
+
 export async function findDonorId(submitterId: string, programId: string) {
   const donor = await findDonorBySubmitterId(submitterId, programId);
   if (!donor) {
@@ -219,27 +225,46 @@ export const getPaginatedClinicalData = async (
   query: PaginatedClinicalQuery,
 ) => {
   if (!programId) throw new Error('Missing programId!');
-  const start = new Date().getTime() / 1000;
 
   const allSchemasWithFields = await dictionaryManager.instance().getSchemasWithFields();
   // Get all donors + records for given entity
   const { donors, totalDonors } = await donorDao.findByPaginatedProgramId(programId, query);
 
   const taskToRun = WorkerTasks.ExtractEntityDataFromDonors;
-  const taskArgs = [donors as Donor[], totalDonors, allSchemasWithFields, query];
+  const taskArgs = [donors as Donor[], totalDonors, allSchemasWithFields, query.entityTypes, query];
 
   // Return paginated data
   const data = await runTaskInWorkerThread<{ clinicalEntities: ClinicalEntityData[] }>(
     taskToRun,
     taskArgs,
   );
 
-  const end = new Date().getTime() / 1000;
-  L.debug(`getPaginatedClinicalData took ${end - start}s`);
-
   return data;
 };
 
+export const getDonorEntityData = async (donorIds: number[]) => {
+  const allSchemasWithFields = await dictionaryManager.instance().getSchemasWithFields();
+  // Get all donors + records for given entity
+  const donors = await getDonorsByIds(donorIds);
+  const totalDonors = donors.length;
+
+  const paginationQuery: PaginationQuery = {
+    page: 0,
+    sort: 'donorId',
+    pageSize: totalDonors,
+  };
+
+  const taskToRun = WorkerTasks.ExtractEntityDataFromDonors;
+  const taskArgs = [donors, donors.length, allSchemasWithFields, allEntityNames, paginationQuery];
+
+  // Return paginated data
+  const data = await runTaskInWorkerThread<{ clinicalEntities: ClinicalEntityData[] }>(
+    taskToRun,
+    taskArgs,
+  );
+  return data.clinicalEntities;
+};
+
 export const getClinicalSearchResults = async (
   programId: string,
   query: ClinicalDonorEntityQuery,

src/clinical/donor-repo.ts

@@ -91,6 +91,7 @@ export interface DonorRepository {
     programId: string,
     query: ClinicalDonorEntityQuery,
   ): Promise<DeepReadonly<{ donors: Donor[] }>>;
+  findByDonorIds(donorIds: number[]): Promise<DeepReadonly<Donor[]>>;
   deleteByProgramId(programId: string): Promise<void>;
   deleteByProgramIdAndDonorIds(programId: string, donorIds: number[]): Promise<void>;
   findByProgramAndSubmitterId(
@@ -392,7 +393,17 @@ export const donorDao: DonorRepository = {
       submitterId: { $in: submitterIds },
       programId: programId,
     });
-    const mapped = result.map((d: DonorDocument) => {
+    const mapped = result.map(d => {
+      return MongooseUtils.toPojo(d) as Donor;
+    });
+    return F(mapped);
+  },
+
+  async findByDonorIds(donorIds: number[]): Promise<DeepReadonly<Donor[]>> {
+    const result = await DonorModel.find({
+      donorId: { $in: donorIds },
+    });
+    const mapped = result.map(d => {
       return MongooseUtils.toPojo(d) as Donor;
     });
     return F(mapped);

src/clinical/service-worker-thread/tasks.ts

@@ -19,7 +19,11 @@
 
 import { DeepReadonly } from 'deep-freeze';
 import _, { isEmpty } from 'lodash';
-import { ClinicalEntitySchemaNames, aliasEntityNames } from '../../common-model/entities';
+import {
+  ClinicalEntitySchemaNames,
+  EntityAlias,
+  aliasEntityNames,
+} from '../../common-model/entities';
 import {
   calculateSpecimenCompletionStats,
   dnaSampleFilter,
@@ -29,7 +33,7 @@ import {
   getSampleRegistrationDataFromDonor,
 } from '../../common-model/functions';
 import { notEmpty } from '../../utils';
-import { PaginatedClinicalQuery, ClinicalDonorEntityQuery } from '../clinical-service';
+import { ClinicalDonorEntityQuery, PaginationQuery } from '../clinical-service';
 import {
   ClinicalEntityData,
   ClinicalInfo,
@@ -117,13 +121,14 @@ const mapEntityDocuments = (
   entity: EntityClinicalInfo,
   donorCount: number,
   schemas: any,
-  query: PaginatedClinicalQuery,
+  entityTypes: EntityAlias[],
+  paginationQuery: PaginationQuery,
   completionStats: CompletionDisplayRecord[],
 ) => {
   const { entityName, results } = entity;
 
   // Filter, Paginate + Sort
-  const { page, pageSize = results.length, sort, entityTypes } = query;
+  const { page, pageSize = results.length, sort } = paginationQuery;
   const relevantSchemaWithFields = schemas.find((s: any) => s.name === entityName);
   const entityInQuery = isEntityInQuery(entityName, entityTypes);
 
@@ -239,21 +244,20 @@ function extractEntityDataFromDonors(
   donors: Donor[],
   totalDonors: number,
   schemasWithFields: any,
-  query: PaginatedClinicalQuery,
+  entityTypes: EntityAlias[],
+  paginationQuery: PaginationQuery,
 ) {
   let clinicalEntityData: EntityClinicalInfo[] = [];
 
   donors.forEach(d => {
     Object.values(ClinicalEntitySchemaNames).forEach(entity => {
-      const isQueriedEntity = isEntityInQuery(entity, query.entityTypes);
-      const isRelatedEntity = getRequiredDonorFieldsForEntityTypes(query.entityTypes).includes(
-        entity,
-      );
+      const isQueriedEntity = isEntityInQuery(entity, entityTypes);
+      const isRelatedEntity = getRequiredDonorFieldsForEntityTypes(entityTypes).includes(entity);
       const requiresSampleRegistration =
         entity === ClinicalEntitySchemaNames.REGISTRATION &&
-        (query.entityTypes.includes('donor') || query.entityTypes.includes('sampleRegistration'));
+        (entityTypes.includes('donor') || entityTypes.includes('sampleRegistration'));
       const requiresSpecimens =
-        entity === ClinicalEntitySchemaNames.SPECIMEN && query.entityTypes.includes('donor');
+        entity === ClinicalEntitySchemaNames.SPECIMEN && entityTypes.includes('donor');
 
       const isRequiredEntity =
         isQueriedEntity || isRelatedEntity || requiresSampleRegistration || requiresSpecimens;
@@ -287,7 +291,14 @@ function extractEntityDataFromDonors(
 
   const clinicalEntities: ClinicalEntityData[] = clinicalEntityData
     .map((entity: EntityClinicalInfo) =>
-      mapEntityDocuments(entity, totalDonors, schemasWithFields, query, completionStats),
+      mapEntityDocuments(
+        entity,
+        totalDonors,
+        schemasWithFields,
+        entityTypes,
+        paginationQuery,
+        completionStats,
+      ),
     )
     .filter(notEmpty);
 

src/common-model/entities.ts

@@ -80,7 +80,7 @@ export const aliasEntityNames: Record<ClinicalEntitySchemaNames, EntityAlias> =
   biomarker: 'biomarker',
 };
 
-export const queryEntityNames = Object.values(aliasEntityNames);
+export const allEntityNames = Object.values(aliasEntityNames);
 
 export interface ClinicalEntityErrorRecord extends dictionaryEntities.SchemaValidationError {
   entityName: ClinicalEntitySchemaNames;

src/logger.ts

@@ -40,13 +40,10 @@ export interface Logger {
 }
 
 const winstonLogger = winston.createLogger(logConfiguration);
-if (process.env.LOG_LEVEL == 'debug') {
+if (process.env.LOG_LEVEL === 'debug') {
   console.log('logger configured: ', winstonLogger);
 }
 export const loggerFor = (fileName: string): Logger => {
-  if (process.env.LOG_LEVEL == 'debug') {
-    console.debug('creating logger for', fileName);
-  }
   const source = fileName.substring(fileName.indexOf('argo-clinical'));
   return {
     error: (msg: string, err: Error): void => {

src/resources/swagger.yaml

@@ -842,44 +842,43 @@ paths:
         '200':
           description: 'returns new entity exception after deletion'
 
-  /clinical/donors:
-    get:
-      deprecated: true
+  /clinical/donors/tsv:
+    post:
       tags:
-        - Deprecated
-      summary: Get all program donors from db
-      parameters:
-        - $ref: '#/components/parameters/QueryProgramId'
+        - Clinical Data
+      summary: Download TSVs for donors by ID.
+      description: This request downloads all clinical data for donors rquested by ID. The response is a zip containing all clinical data as TSVs. Requires full system read permission to access.
+      requestBody:
+        description: JSON object with list of donor IDs
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                donorIds:
+                  description: List of donor IDs, as numbers, to be included in request. At least one Donor ID minimum is required.
+                  type: array
+                  items:
+                    type: numbers
+                  default: []
       responses:
         '401':
           $ref: '#/components/responses/UnauthorizedError'
         '403':
           $ref: '#/components/responses/ForbiddenError'
+        '400':
+          description: 'User input error. Either the request body was invalid, or some of the requested Donor IDs were not found.'
         '500':
           $ref: '#/components/responses/ServerError'
         '200':
-          description: 'List of donors for the given program'
+          description: 'Zip file with clinical data for requested donors.'
           content:
-            application/json:
+            application/zip:
               schema:
-                type: array
-                items:
-                  type: object
-    delete:
-      tags:
-        - Test
-      summary: Test endpoint to delete program donors in db (disabled in prod)
-      parameters:
-        - $ref: '#/components/parameters/QueryProgramId'
-      responses:
-        '401':
-          $ref: '#/components/responses/UnauthorizedError'
-        '403':
-          $ref: '#/components/responses/ForbiddenError'
-        '500':
-          $ref: '#/components/responses/ServerError'
-        '200':
-          description: 'all donors deleted'
+                type: string
+                format: binary
+
   /clinical/program/{programId}/donor/{donorId}:
     get:
       tags:
@@ -1068,6 +1067,16 @@ paths:
       description: Returns a zip file with commited clinical entity data in tsvs, filtered by request query. Exported data may not be valid with current dictionary.
       parameters:
         - $ref: '#/components/parameters/PathProgramId'
+        - in: query
+          name: page
+          schema:
+            type: integer
+          description: Paginated Results page number to return
+        - in: query
+          name: pageSize
+          schema:
+            type: integer
+          description: Number of paginated results per page
       requestBody:
         description: JSON object containing search filters
         required: true
@@ -1359,8 +1368,6 @@ tags:
     description: APIs to do the icgc import
   - name: Admin
     description: Admin operations collected for convenience
-  - name: Test
-    description: Protected endpoints that run in dev only, not in prod.
   - name: Deprecated
     description: Deprecated enpoints
 components:

src/routes/data.ts

@@ -45,10 +45,10 @@ router.post(
   wrapAsync(clinicalApi.getProgramClinicalSearchResults),
 );
 router.post('/program/:programId/clinical-errors', wrapAsync(clinicalApi.getProgramClinicalErrors));
-router.post(
-  '/program/:programId/clinical-tsv',
-  wrapAsync(clinicalApi.getSpecificClinicalDataAsTsvsInZip),
-);
+router.post('/program/:programId/clinical-tsv', wrapAsync(clinicalApi.getDonorDataByIdAsTsvsInZip));
+
+// Download TSVs for Donors by Donor ID
+router.post('/donors/tsv', wrapAsync(clinicalApi.getDonorDataByIdAsTsvsInZip));
 
 // Get TSV Data
 router.get(