Merge pull request #1188 from icgc-argo/rc/1.87.2

release a fix for issue #1186

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "argo-clinical",
-	"version": "1.87.1",
+	"version": "1.87.2",
 	"description": "Clinical submission system and repo.",
 	"scripts": {
 		"start": "npm run serve",
@@ -97,13 +97,13 @@
 		"typescript": "^5.0.0"
 	},
 	"dependencies": {
-		"@apollo/subgraph": "2.5.2",
-		"apollo-server-core": "^3.12.0",
 		"@apollo/server": "4.0.0",
-		"@overturebio-stack/lectern-client": "1.4.0",
-		"@types/mongoose-paginate-v2": "^1.3.11",
+		"@apollo/subgraph": "2.5.2",
 		"@icgc-argo/ego-token-utils": "^8.2.0",
+		"@overturebio-stack/lectern-client": "^1.5.0",
+		"@types/mongoose-paginate-v2": "^1.3.11",
 		"adm-zip": "^0.4.16",
+		"apollo-server-core": "^3.12.0",
 		"async": "^3.0.1",
 		"bcrypt-nodejs": "^0.0.3",
 		"bluebird": "^3.5.5",

src/clinical/api/clinical-api.ts

@@ -19,7 +19,7 @@
 
 import { Request, Response } from 'express';
 import * as service from '../clinical-service';
-import { ClinicalDataQuery } from '../clinical-service';
+import { ClinicalDataQuery } from '../types';
 import { getExceptionManifestRecords } from '../../submission/exceptions/exceptions';
 import { ExceptionManifestRecord } from '../../exception/exception-manifest/types';
 import {

src/clinical/clinical-service.ts

@@ -27,7 +27,6 @@ import {
 	ClinicalEntityErrorRecord,
 	ClinicalEntitySchemaNames,
 	ClinicalErrorsResponseRecord,
-	EntityAlias,
 	aliasEntityNames,
 	allEntityNames,
 } from '../common-model/entities';
@@ -52,40 +51,16 @@ import { ClinicalEntityData, Donor, Sample } from './clinical-entities';
 import { DONOR_DOCUMENT_FIELDS, donorDao } from './donor-repo';
 import { runTaskInWorkerThread } from './service-worker-thread/runner';
 import { WorkerTasks } from './service-worker-thread/tasks';
-import { CompletionState } from './api/types';
+import {
+	ClinicalDataQuery,
+	ClinicalDataSortType,
+	ClinicalDataSortTypes,
+	ClinicalDonorEntityQuery,
+	PaginationQuery,
+} from './types';
 
 const L = loggerFor(__filename);
 
-// Base type for Clinical Data Queries
-export type ClinicalDonorEntityQuery = {
-	donorIds: number[];
-	submitterDonorIds: string[];
-	entityTypes: EntityAlias[];
-};
-
-export type PaginationQuery = {
-	page: number;
-	pageSize?: number;
-	sort: string;
-};
-
-type ClinicalDataPaginatedQuery = ClinicalDonorEntityQuery & PaginationQuery;
-
-export type ClinicalDataQuery = ClinicalDataPaginatedQuery & {
-	completionState?: {};
-};
-
-// GQL Query Arguments
-// Submitted Data Table, SearchBar, Sidebar, etc.
-export type ClinicalDataApiFilters = ClinicalDataPaginatedQuery & {
-	completionState?: CompletionState;
-};
-
-export type ClinicalDataVariables = {
-	programShortName: string;
-	filters: ClinicalDataApiFilters;
-};
-
 export async function updateDonorSchemaMetadata(
 	donor: DeepReadonly<Donor>,
 	migrationId: string,
@@ -231,8 +206,31 @@ export const getPaginatedClinicalData = async (programId: string, query: Clinica
 	// Get all donors + records for given entity
 	const { donors, totalDonors } = await donorDao.findByPaginatedProgramId(programId, query);
 
+	const donorIds = donors.map((donor) => donor.donorId);
+
+	const isDefaultDonorSort = query.sort.includes('completionStats.coreCompletionPercentage');
+	const isInvalidSort = query.sort.includes('schemaMetadata.isValid');
+
+	const clinicalErrors = isInvalidSort
+		? (await getClinicalErrors(programId, donorIds)).clinicalErrors
+		: [];
+
+	const sortType: ClinicalDataSortType = isDefaultDonorSort
+		? ClinicalDataSortTypes.defaultDonor
+		: isInvalidSort
+		? ClinicalDataSortTypes.invalidEntity
+		: ClinicalDataSortTypes.columnSort;
+
 	const taskToRun = WorkerTasks.ExtractEntityDataFromDonors;
-	const taskArgs = [donors as Donor[], totalDonors, allSchemasWithFields, query.entityTypes, query];
+	const taskArgs = [
+		donors as Donor[],
+		totalDonors,
+		allSchemasWithFields,
+		query.entityTypes,
+		query,
+		sortType,
+		clinicalErrors,
+	];
 
 	// Return paginated data
 	const data = await runTaskInWorkerThread<{ clinicalEntities: ClinicalEntityData[] }>(

src/clinical/donor-repo.ts

@@ -18,7 +18,7 @@
  */
 
 import { Donor } from './clinical-entities';
-import { ClinicalDataQuery, ClinicalDonorEntityQuery } from './clinical-service';
+import { ClinicalDataQuery, ClinicalDonorEntityQuery } from './types';
 import { getRequiredDonorFieldsForEntityTypes } from '../common-model/functions';
 import mongoose, { PaginateModel } from 'mongoose';
 import mongoosePaginate from 'mongoose-paginate-v2';

src/clinical/service-worker-thread/tasks.ts

@@ -21,6 +21,7 @@ import { DeepReadonly } from 'deep-freeze';
 import _, { isEmpty } from 'lodash';
 import {
 	ClinicalEntitySchemaNames,
+	ClinicalErrorsResponseRecord,
 	EntityAlias,
 	aliasEntityNames,
 } from '../../common-model/entities';
@@ -33,7 +34,12 @@ import {
 	getSampleRegistrationDataFromDonor,
 } from '../../common-model/functions';
 import { notEmpty } from '../../utils';
-import { ClinicalDonorEntityQuery, PaginationQuery } from '../clinical-service';
+import {
+	ClinicalDonorEntityQuery,
+	ClinicalDataSortType,
+	ClinicalDataSortTypes,
+	PaginationQuery,
+} from '../types';
 import {
 	ClinicalEntityData,
 	ClinicalInfo,
@@ -55,32 +61,24 @@ const DONOR_ID_FIELD = 'donor_id';
 const isEntityInQuery = (entityName: ClinicalEntitySchemaNames, entityTypes: string[]) =>
 	entityTypes.includes(aliasEntityNames[entityName]);
 
-// Main Sort Function
-const sortDocs = (
+// Base Sort Function Wrapper
+function sortDocs<SortArgs>(
 	sortQuery: string,
-	entityName: string,
-	completionStats: CompletionDisplayRecord[],
-) => (currentRecord: ClinicalInfo, nextRecord: ClinicalInfo) => {
-	// Sort Value: 0 order is Unchanged, -1 Current lower index than Next, +1 Current higher index than Next
-	let order = 0;
-	const isDescending = sortQuery.startsWith('-');
-	const isDefaultSort =
-		entityName === ClinicalEntitySchemaNames.DONOR &&
-		sortQuery.includes('completionStats.coreCompletionPercentage');
-
-	const queryKey = isDescending ? sortQuery.split('-')[1] : sortQuery;
-	const key = queryKey === 'donorId' ? 'donor_id' : queryKey;
-
-	if (isDefaultSort) {
-		order = sortDonorRecordsByCompletion(currentRecord, nextRecord, completionStats);
-	} else {
-		order = sortRecordsByColumn(currentRecord, nextRecord, key);
-	}
+	sortArgs: SortArgs,
+	sortFunction: (currentRecord: ClinicalInfo, nextRecord: ClinicalInfo, args: SortArgs) => number,
+) {
+	return (currentRecord: ClinicalInfo, nextRecord: ClinicalInfo) => {
+		// Sort Value: 0 order is Unchanged, -1 Current lower index than Next, +1 Current higher index than Next
+		let order = 0;
+		const isDescending = sortQuery.startsWith('-');
 
-	order = isDescending ? -order : order;
+		order = sortFunction(currentRecord, nextRecord, sortArgs);
 
-	return order;
-};
+		order = isDescending ? -order : order;
+
+		return order;
+	};
+}
 
 // Sort Clinically Incomplete donors to top (sorted by donorId at DB level)
 const sortDonorRecordsByCompletion = (
@@ -116,6 +114,46 @@ const sortRecordsByColumn = (
 	return valueSort;
 };
 
+// Sort Invalid Records to Top
+const sortInvalidRecords = (
+	errors: ClinicalErrorsResponseRecord[],
+	records: ClinicalInfo[],
+	entityName: ClinicalEntitySchemaNames,
+) => {
+	const entityErrors = errors.filter((errorRecord) => errorRecord.entityName === entityName);
+	const errorIds = new Set(entityErrors.map((error) => error.donorId));
+
+	const validRecords: ClinicalInfo[] = [];
+	const invalidRecords: ClinicalInfo[] = [];
+
+	records.forEach((record) => {
+		if (typeof record.donor_id === 'number') {
+			if (!errorIds.has(record.donor_id)) {
+				validRecords.push(record);
+			} else {
+				const currentRecordIsInvalid = entityErrors.find((errorRecord) => {
+					const idValid = errorRecord.donorId === record.donor_id;
+					const recordValid = errorRecord.errors.some((error) => {
+						const recordValue = record[error.fieldName];
+						const errorValue = Array.isArray(error.info.value)
+							? error.info.value[0]
+							: error.info.value;
+						return recordValue === errorValue;
+					});
+					return idValid && recordValid;
+				});
+				if (currentRecordIsInvalid) {
+					invalidRecords.push(record);
+				} else {
+					validRecords.push(record);
+				}
+			}
+		}
+	});
+
+	return [...invalidRecords, ...validRecords];
+};
+
 // Formats + Organizes Clinical Data
 const mapEntityDocuments = (
 	entity: EntityClinicalInfo,
@@ -124,6 +162,8 @@ const mapEntityDocuments = (
 	entityTypes: EntityAlias[],
 	paginationQuery: PaginationQuery,
 	completionStats: CompletionDisplayRecord[],
+	sortType: ClinicalDataSortType,
+	errors: ClinicalErrorsResponseRecord[],
 ): ClinicalEntityData | undefined => {
 	const { entityName, results } = entity;
 
@@ -137,7 +177,26 @@ const mapEntityDocuments = (
 	}
 
 	const totalDocs = entityName === ClinicalEntitySchemaNames.DONOR ? donorCount : results.length;
-	let records = results.sort(sortDocs(sort, entityName, completionStats));
+
+	let records = results;
+
+	switch (sortType) {
+		case ClinicalDataSortTypes.defaultDonor: {
+			records = results.sort(sortDocs(sort, completionStats, sortDonorRecordsByCompletion));
+			break;
+		}
+		case ClinicalDataSortTypes.invalidEntity: {
+			records = sortInvalidRecords(errors, results, entityName);
+			break;
+		}
+		// Column Sort is the default, fallback here is intentional
+		case ClinicalDataSortTypes.columnSort:
+		default: {
+			const sortKey = sort[0] === '-' ? sort.split('-')[1] : sort;
+			const key = sortKey === 'donorId' ? DONOR_ID_FIELD : sortKey;
+			records = results.sort(sortDocs(sort, key, sortRecordsByColumn));
+		}
+	}
 
 	if (records.length > pageSize) {
 		// Manual Pagination
@@ -254,6 +313,8 @@ function extractEntityDataFromDonors(
 	schemasWithFields: any,
 	entityTypes: EntityAlias[],
 	paginationQuery: PaginationQuery,
+	sortType: ClinicalDataSortType,
+	errors: ClinicalErrorsResponseRecord[],
 ) {
 	let clinicalEntityData: EntityClinicalInfo[] = [];
 
@@ -312,6 +373,8 @@ function extractEntityDataFromDonors(
 				entityTypes,
 				paginationQuery,
 				completionStats,
+				sortType,
+				errors,
 			),
 		)
 		.filter(notEmpty);

src/clinical/types.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved
+ *
+ * This program and the accompanying materials are made available under the terms of
+ * the GNU Affero General Public License v3.0. You should have received a copy of the
+ * GNU Affero General Public License along with this program.
+ *  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import { EntityAlias } from '../common-model/entities';
+import { CompletionState } from './api/types';
+import { Values } from '../utils/objectTypes';
+
+// Types Specific to Clinical Service and Related Tasks
+
+// Base type for Clinical Data Queries
+export type ClinicalDonorEntityQuery = {
+	donorIds: number[];
+	submitterDonorIds: string[];
+	entityTypes: EntityAlias[];
+};
+
+// Types related to sorting, filtering, pagination, etc
+export type PaginationQuery = {
+	page: number;
+	pageSize?: number;
+	sort: string;
+};
+
+export type ClinicalDataPaginatedQuery = ClinicalDonorEntityQuery & PaginationQuery;
+
+export type ClinicalDataQuery = ClinicalDataPaginatedQuery & {
+	completionState?: {};
+};
+
+export const ClinicalDataSortTypes = {
+	defaultDonor: 'defaultDonor',
+	invalidEntity: 'invalidEntity',
+	columnSort: 'columnSort',
+};
+
+export type ClinicalDataSortType = Values<typeof ClinicalDataSortTypes>;
+
+// GQL Query Arguments
+// Submitted Data Table, SearchBar, Sidebar, etc.
+export type ClinicalDataApiFilters = ClinicalDataPaginatedQuery & {
+	completionState?: CompletionState;
+};
+
+export type ClinicalDataVariables = {
+	programShortName: string;
+	filters: ClinicalDataApiFilters;
+};