SIMSBIOHUB-483: Paginate the Projects lists endpoint (#1212)

* Added new environment variables used to the set number of seed projects and surveys to generate: NUM_SEED_PROJECTS and NUM_SEED_SURVEYS_PER_PROJECT.
* Removed frontend code associated with creating, editing and deleting project drafts.
* Optimized SQL queries used to fetch the projects list
* Removed the project submission status from the projects list response
* Removed the project submission status banner
* Modified the list projects endpoint to support server-side pagination

api/.pipeline/templates/api.dc.yaml

@@ -269,6 +269,10 @@ objects:
                   value: '5432'
                 - name: PROJECT_SEEDER_USER_IDENTIFIER
                   value: ${PROJECT_SEEDER_USER_IDENTIFIER}
+                - name: NUM_SEED_PROJECTS
+                  value: ${NUM_SEED_PROJECTS}
+                - name: NUM_SEED_SURVEYS_PER_PROJECT
+                  value: ${NUM_SEED_SURVEYS_PER_PROJECT}
                 # Keycloak
                 - name: KEYCLOAK_HOST
                   value: ${KEYCLOAK_HOST}

api/src/models/project-view.ts

@@ -35,8 +35,7 @@ export type ProjectData = z.infer<typeof ProjectData>;
 
 export const ProjectListData = z.object({
   project_id: z.number(),
-  uuid: z.string().uuid(),
-  project_name: z.string(),
+  name: z.string(),
   project_programs: z.array(z.number()).default([]),
   regions: z.array(z.string()).default([]),
   start_date: z.string(),

api/src/openapi/schemas/pagination.ts

@@ -0,0 +1,86 @@
+import { SchemaObject } from 'ajv';
+
+/**
+ * API schema used to assert pagination query paramaters
+ * for paginated data requests.
+ */
+export const paginationRequestQueryParamSchema: any[] = [
+  {
+    in: 'query',
+    name: 'page',
+    required: false,
+    schema: {
+      type: 'integer',
+      minimum: 1,
+      description: 'The current page number to be fetched'
+    }
+  },
+  {
+    in: 'query',
+    name: 'limit',
+    required: false,
+    schema: {
+      type: 'integer',
+      minimum: 1,
+      maximum: 100,
+      description: 'The number of records to show per page'
+    }
+  },
+  {
+    in: 'query',
+    name: 'sort',
+    required: false,
+    description: `The column to be sorted on, e.g. 'name'`,
+    schema: {
+      type: 'string'
+    }
+  },
+  {
+    in: 'query',
+    name: 'order',
+    required: false,
+    description: 'The order of the sort, i.e. asc or desc',
+    schema: {
+      type: 'string',
+      enum: ['asc', 'desc']
+    }
+  }
+];
+
+/**
+ * API schema to assert pagination information for paginated data
+ * responses.
+ */
+export const paginationResponseSchema: SchemaObject = {
+  type: 'object',
+  required: ['total', 'current_page', 'last_page'],
+  properties: {
+    total: {
+      type: 'integer',
+      description: 'The total number of records belonging to the collection'
+    },
+    per_page: {
+      type: 'integer',
+      minimum: 1,
+      description: 'The number of records shown per page'
+    },
+    current_page: {
+      type: 'integer',
+      description: 'The current page being fetched'
+    },
+    last_page: {
+      type: 'integer',
+      minimum: 1,
+      description: 'The total number of pages'
+    },
+    sort: {
+      type: 'string',
+      description: 'The column that is being sorted on'
+    },
+    order: {
+      type: 'string',
+      enum: ['asc', 'desc'],
+      description: 'The sort order of the response'
+    }
+  }
+};

api/src/paths/project/list.test.ts

@@ -4,9 +4,9 @@ import { describe } from 'mocha';
 import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
 import { SYSTEM_ROLE } from '../../constants/roles';
+import { COMPLETION_STATUS } from '../../constants/status';
 import * as db from '../../database/db';
 import { HTTPError } from '../../errors/http-error';
-import { PublishStatus } from '../../repositories/history-publish-repository';
 import * as authorization from '../../request-handlers/security/authorization';
 import { ProjectService } from '../../services/project-service';
 import { getMockDBConnection } from '../../__mocks__/db';
@@ -33,6 +33,11 @@ describe('list', () => {
       }
     } as any;
 
+    sampleReq.query = {
+      page: '1',
+      limit: '10'
+    };
+
     let actualResult: any = null;
 
     const sampleRes = {
@@ -58,12 +63,23 @@ describe('list', () => {
       });
       sinon.stub(authorization, 'userHasValidRole').returns(true);
       sinon.stub(ProjectService.prototype, 'getProjectList').resolves([]);
+      sinon.stub(ProjectService.prototype, 'getProjectCount').resolves(0);
 
       const result = list.getProjectList();
 
       await result(sampleReq, sampleRes as any, (null as unknown) as any);
 
-      expect(actualResult).to.eql([]);
+      expect(actualResult).to.eql({
+        pagination: {
+          current_page: 1,
+          last_page: 1,
+          total: 0,
+          sort: undefined,
+          order: undefined,
+          per_page: 10
+        },
+        projects: []
+      });
     });
 
     it('returns an array of projects', async () => {
@@ -75,34 +91,45 @@ describe('list', () => {
       });
       sinon.stub(authorization, 'userHasValidRole').returns(true);
 
-      const expectedResponse1 = [
+      const getProjectListStub = sinon.stub(ProjectService.prototype, 'getProjectList').resolves([
         {
-          projectData: {
-            id: 1,
-            name: 'myproject',
-            project_programs: [1],
-            start_date: '2022-02-02',
-            end_date: null,
-            completion_status: 'done'
-          },
-          projectSupplementaryData: { publishStatus: 'SUBMITTED' }
+          project_id: 1,
+          name: 'myproject',
+          project_programs: [1],
+          start_date: '2022-02-02',
+          end_date: null,
+          regions: [],
+          completion_status: COMPLETION_STATUS.COMPLETED
         }
-      ];
-
-      const getProjectListStub = sinon
-        .stub(ProjectService.prototype, 'getProjectList')
-        .resolves([expectedResponse1[0].projectData]);
-      const getSurveyHasUnpublishedContentStub = sinon
-        .stub(ProjectService.prototype, 'projectPublishStatus')
-        .resolves(PublishStatus.SUBMITTED);
+      ]);
+      sinon.stub(ProjectService.prototype, 'getProjectCount').resolves(1);
 
       const result = list.getProjectList();
 
       await result(sampleReq, (sampleRes as unknown) as any, (null as unknown) as any);
 
-      expect(actualResult).to.eql(expectedResponse1);
+      expect(actualResult).to.eql({
+        pagination: {
+          current_page: 1,
+          last_page: 1,
+          total: 1,
+          sort: undefined,
+          order: undefined,
+          per_page: 10
+        },
+        projects: [
+          {
+            project_id: 1,
+            name: 'myproject',
+            project_programs: [1],
+            start_date: '2022-02-02',
+            end_date: null,
+            regions: [],
+            completion_status: COMPLETION_STATUS.COMPLETED
+          }
+        ]
+      });
       expect(getProjectListStub).to.be.calledOnce;
-      expect(getSurveyHasUnpublishedContentStub).to.be.calledOnce;
     });
 
     it('catches error, calls rollback, and re-throws error', async () => {

api/src/paths/project/list.ts

@@ -3,9 +3,11 @@ import { Operation } from 'express-openapi';
 import { SYSTEM_ROLE } from '../../constants/roles';
 import { getDBConnection } from '../../database/db';
 import { IProjectAdvancedFilters } from '../../models/project-view';
+import { paginationRequestQueryParamSchema, paginationResponseSchema } from '../../openapi/schemas/pagination';
 import { authorizeRequestHandler, userHasValidRole } from '../../request-handlers/security/authorization';
 import { ProjectService } from '../../services/project-service';
 import { getLogger } from '../../utils/logger';
+import { ApiPaginationOptions } from '../../zod-schema/pagination';
 
 const defaultLog = getLogger('paths/projects');
 
@@ -30,6 +32,7 @@ GET.apiDoc = {
       Bearer: []
     }
   ],
+  parameters: [...paginationRequestQueryParamSchema],
   requestBody: {
     description: 'Project list search filter criteria object.',
     content: {
@@ -47,7 +50,7 @@ GET.apiDoc = {
             project_programs: {
               type: 'array',
               items: {
-                type: 'number'
+                type: 'integer'
               },
               nullable: true
             },
@@ -59,10 +62,11 @@ GET.apiDoc = {
               type: 'string',
               nullable: true
             },
+            // TODO rename this to imply ITIS TSN filtering
             species: {
               type: 'array',
               items: {
-                type: 'number'
+                type: 'integer'
               }
             }
           }
@@ -76,34 +80,33 @@ GET.apiDoc = {
       content: {
         'application/json': {
           schema: {
-            type: 'array',
-            items: {
-              title: 'Survey get response object, for view purposes',
-              type: 'object',
-              required: ['projectData', 'projectSupplementaryData'],
-              properties: {
-                projectData: {
+            type: 'object',
+            required: ['projects', 'pagination'],
+            properties: {
+              projects: {
+                type: 'array',
+                items: {
                   type: 'object',
                   required: [
-                    'id',
+                    'project_id',
                     'name',
                     'project_programs',
+                    'completion_status',
                     'start_date',
                     'end_date',
-                    'completion_status',
                     'regions'
                   ],
                   properties: {
-                    id: {
-                      type: 'number'
+                    project_id: {
+                      type: 'integer'
                     },
                     name: {
                       type: 'string'
                     },
                     project_programs: {
                       type: 'array',
                       items: {
-                        type: 'number'
+                        type: 'integer'
                       }
                     },
                     start_date: {
@@ -122,18 +125,9 @@ GET.apiDoc = {
                       }
                     }
                   }
-                },
-                projectSupplementaryData: {
-                  type: 'object',
-                  required: ['publishStatus'],
-                  properties: {
-                    publishStatus: {
-                      type: 'string',
-                      enum: ['NO_DATA', 'UNSUBMITTED', 'SUBMITTED']
-                    }
-                  }
                 }
-              }
+              },
+              pagination: { ...paginationResponseSchema }
             }
           }
         }
@@ -164,6 +158,13 @@ GET.apiDoc = {
  */
 export function getProjectList(): RequestHandler {
   return async (req, res) => {
+    defaultLog.debug({ label: 'getProjectList' });
+
+    const page: number | undefined = req.query.page ? Number(req.query.page) : undefined;
+    const limit: number | undefined = req.query.limit ? Number(req.query.limit) : undefined;
+    const order: 'asc' | 'desc' | undefined = req.query.order ? (String(req.query.order) as 'asc' | 'desc') : undefined;
+    const sort: string | undefined = req.query.sort ? String(req.query.sort) : undefined;
+
     const connection = getDBConnection(req['keycloak_token']);
 
     try {
@@ -178,21 +179,27 @@ export function getProjectList(): RequestHandler {
 
       const projectService = new ProjectService(connection);
 
-      const projects = await projectService.getProjectList(isUserAdmin, systemUserId, filterFields);
+      const paginationOptions: ApiPaginationOptions | undefined =
+        limit !== undefined && page !== undefined ? { limit, page, sort, order } : undefined;
 
-      const projectListWithStatus = await Promise.all(
-        projects.map(async (project: any) => {
-          const status = await projectService.projectPublishStatus(project.id);
-          return {
-            projectData: project,
-            projectSupplementaryData: { publishStatus: status }
-          };
-        })
-      );
+      const projects = await projectService.getProjectList(isUserAdmin, systemUserId, filterFields, paginationOptions);
+      const projectsTotalCount = await projectService.getProjectCount(isUserAdmin, systemUserId);
+
+      const response = {
+        projects,
+        pagination: {
+          total: projectsTotalCount,
+          per_page: limit,
+          current_page: page ?? 1,
+          last_page: limit ? Math.max(1, Math.ceil(projectsTotalCount / limit)) : 1,
+          sort,
+          order
+        }
+      };
 
       await connection.commit();
 
-      return res.status(200).json(projectListWithStatus);
+      return res.status(200).json(response);
     } catch (error) {
       defaultLog.error({ label: 'getProjectList', message: 'error', error });
       throw error;