Feat/#1186 disable cross field validation for survival time (#1214)

* add program exception check * add check for exception in specimen validation * fix conditional search param object creation * add exception lookup check to treatment * add follow_up exception check * add entity exception check also * fix type error * handling if program_id is undefined * Skip time interval checks when `survival_time` value is missing * cleanup exception code * fix TS build error * remove specimen_acquisition_interval from not info validation info unit test * remove survival_time assertion on follow up unit test --------- Co-authored-by: Ciaran Schutte <[email protected]> Co-authored-by: Jon Eubank <[email protected]>
icgc-argo · Nov 18, 2024 · c4aa428 · c4aa428
1 parent 714dc1c
commit c4aa428
Show file tree

Hide file tree

Showing 8 changed files with 187 additions and 165 deletions.
diff --git a/src/exception/property-exceptions/repo/entity.ts b/src/exception/property-exceptions/repo/entity.ts
@@ -105,11 +105,19 @@ const entityExceptionRepository = {
 		}
 	},
 
-	async find(programId: string): Promise<EntityException | null> {
+	async find(
+		programId: string,
+		optionalSearchParams?: Record<string, string>,
+	): Promise<EntityException | null> {
 		L.debug(`Finding entity exception with program id: ${JSON.stringify(programId)}`);
 		try {
+			const searchParams = {
+				programId,
+				...optionalSearchParams,
+			};
+
 			// first found document, or null
-			const doc = await EntityExceptionModel.findOne({ programId }).lean(true);
+			const doc = await EntityExceptionModel.findOne(searchParams).lean(true);
 			return doc;
 		} catch (e) {
 			L.error('Failed to find program exception', e);

diff --git a/src/exception/property-exceptions/repo/program.ts b/src/exception/property-exceptions/repo/program.ts
@@ -58,10 +58,20 @@ const programExceptionRepository = {
 		}
 	},
 
-	async find(programId: string): Promise<ProgramException | null> {
+	async find(
+		programId: string,
+		optionalSearchParams?: { requested_core_field?: string },
+	): Promise<ProgramException | null> {
 		L.debug(`finding program exception with id: ${JSON.stringify(programId)}`);
 		try {
-			const doc = await ProgramExceptionModel.findOne({ programId }).lean(true);
+			const searchParams: Record<string, string> = {
+				programId,
+			};
+			if (optionalSearchParams?.requested_core_field) {
+				searchParams['exceptions.requested_core_field'] =
+					optionalSearchParams?.requested_core_field;
+			}
+			const doc = await ProgramExceptionModel.findOne(searchParams).lean(true);
 			return doc;
 		} catch (e) {
 			L.error('failed to find program exception', e);

diff --git a/src/submission/exceptions/exceptions.ts b/src/submission/exceptions/exceptions.ts
@@ -44,6 +44,7 @@ import {
 	findDonorsBySubmitterIds,
 } from '../../clinical/clinical-service';
 import { notEmpty } from '../../utils';
+import { SubmittedClinicalRecord } from '../submission-entities';
 
 /**
  * query db for program or entity exceptions
@@ -335,3 +336,43 @@ export async function getExceptionManifestRecords(
 
 	return exceptionManifest;
 }
+
+/**
+ * Boolean check for existence of exceptions for a specific field
+ *
+ * @param record
+ * @param field
+ * @param schema
+ * @returns boolean indicating if an exception exists
+ */
+export const checkForExceptions = async ({
+	record,
+	field,
+	schema,
+}: {
+	record: DeepReadonly<SubmittedClinicalRecord>;
+	field: string;
+	schema: string;
+}): Promise<boolean> => {
+	const programId = record['program_id'] as string;
+
+	if (!programId) return false;
+
+	const programAdditionalSearchParams = { requested_core_field: field };
+
+	const entityAdditionalSearchParams = {
+		[`${schema}.requested_core_field`]: field,
+	};
+
+	// retrieve submitted exceptions for program id (both program level and entity level)
+	const programException = await programExceptionRepository.find(
+		programId,
+		programAdditionalSearchParams,
+	);
+	const entityException = await entityExceptionRepository.find(
+		programId,
+		entityAdditionalSearchParams,
+	);
+
+	return notEmpty(programException) || notEmpty(entityException);
+};
diff --git a/src/submission/validation-clinical/followUp.ts b/src/submission/validation-clinical/followUp.ts
@@ -24,13 +24,102 @@ import {
 	SubmissionValidationOutput,
 	DonorVitalStatusValues,
 } from '../submission-entities';
-import { ClinicalEntitySchemaNames, FollowupFieldsEnum } from '../../common-model/entities';
+import {
+	ClinicalEntitySchemaNames,
+	DonorFieldsEnum,
+	FollowupFieldsEnum,
+} from '../../common-model/entities';
 import { DeepReadonly } from 'deep-freeze';
 import { Donor, Treatment } from '../../clinical/clinical-entities';
 import _ from 'lodash';
 import { getClinicalEntitiesFromDonorBySchemaName } from '../../common-model/functions';
 import { getEntitySubmitterIdFieldName } from '../../common-model/functions';
 import * as utils from './utils';
+
+/**
+ * !Mutates the errors array if an error is found!
+ *
+ * Validates time conflict for `interval_of_followup` compared to the donor's survival_time.
+ * If there is an error we push an error to the `errors` array.
+ *
+ * @param donorDataToValidateWith
+ * @param followUpRecord
+ * @param errors
+ */
+const checkDonorTimeConflict = (
+	donorDataToValidateWith: { [k: string]: any },
+	followUpRecord: DeepReadonly<SubmittedClinicalRecord>,
+	errors: SubmissionValidationError[],
+) => {
+	if (
+		donorDataToValidateWith.donorVitalStatus === DonorVitalStatusValues.deceased &&
+		donorDataToValidateWith.donorSurvivalTime <
+			followUpRecord[FollowupFieldsEnum.interval_of_followup]
+	) {
+		errors.push(
+			utils.buildSubmissionError(
+				followUpRecord,
+				DataValidationErrors.FOLLOW_UP_DONOR_TIME_CONFLICT,
+				FollowupFieldsEnum.interval_of_followup,
+				{},
+			),
+		);
+	}
+};
+
+/**
+ * !Mutates the errors array if an error is found!
+ *
+ * Validates time conflict for `treatment_start_interval` compared to the donor's survival_time.
+ * If there is an error we push an error to the `errors` array.
+ *
+ * @param followUpRecord
+ * @param treatment
+ * @param errors
+ * @returns
+ */
+function checkTreatmentTimeConflict(
+	followUpRecord: DeepReadonly<SubmittedClinicalRecord>,
+	treatment: DeepReadonly<Treatment>,
+	errors: SubmissionValidationError[],
+) {
+	// A follow up may or may not be associated with treatment
+	if (treatment == undefined) return;
+
+	if (
+		followUpRecord.interval_of_followup &&
+		treatment.clinicalInfo &&
+		treatment.clinicalInfo.treatment_start_interval &&
+		followUpRecord.interval_of_followup <= treatment.clinicalInfo.treatment_start_interval
+	) {
+		errors.push(
+			utils.buildSubmissionError(
+				followUpRecord,
+				DataValidationErrors.FOLLOW_UP_CONFLICING_INTERVAL,
+				FollowupFieldsEnum.interval_of_followup,
+				[],
+			),
+		);
+	}
+}
+
+function getExistingFollowUp(
+	existingDonor: DeepReadonly<Donor>,
+	record: DeepReadonly<SubmittedClinicalRecord>,
+) {
+	if (existingDonor.followUps) {
+		return getClinicalEntitiesFromDonorBySchemaName(
+			existingDonor,
+			ClinicalEntitySchemaNames.FOLLOW_UP,
+		).find(
+			(ci) =>
+				ci[FollowupFieldsEnum.submitter_follow_up_id] ==
+				record[FollowupFieldsEnum.submitter_follow_up_id],
+		);
+	}
+	return undefined;
+}
+
 export const validate = async (
 	followUpRecord: DeepReadonly<SubmittedClinicalRecord>,
 	existentDonor: DeepReadonly<Donor>,
@@ -62,47 +151,25 @@ export const validate = async (
 		false,
 	);
 
-	// Follow_up.interval_of_followup must be less than Donor.survival_time:
-	const donorDataToValidateWith = utils.getDataFromDonorRecordOrDonor(
+	const donorDataToValidateWith = utils.getSurvivalDataFromDonor(
 		followUpRecord,
 		mergedDonor,
-		errors,
 		FollowupFieldsEnum.interval_of_followup,
 	);
 
-	const checkDonorTimeConflict = (
-		donorDataToValidateWith: { [k: string]: any },
-		followUpRecord: DeepReadonly<SubmittedClinicalRecord>,
-		errors: SubmissionValidationError[],
-	) => {
-		if (
-			donorDataToValidateWith.donorVitalStatus === DonorVitalStatusValues.deceased &&
-			donorDataToValidateWith.donorSurvivalTime <
-				followUpRecord[FollowupFieldsEnum.interval_of_followup]
-		) {
-			errors.push(
-				utils.buildSubmissionError(
-					followUpRecord,
-					DataValidationErrors.FOLLOW_UP_DONOR_TIME_CONFLICT,
-					FollowupFieldsEnum.interval_of_followup,
-					{},
-				),
-			);
-		}
-	};
-
-	if (donorDataToValidateWith) {
-		checkDonorTimeConflict(donorDataToValidateWith, followUpRecord, errors);
-	}
-
 	const entitySubmitterIdField = getEntitySubmitterIdFieldName(ClinicalEntitySchemaNames.TREATMENT);
 	const treatment = utils.getRelatedEntityByFK(
 		ClinicalEntitySchemaNames.TREATMENT,
 		followUpRecord[entitySubmitterIdField] as string,
 		mergedDonor,
 	) as DeepReadonly<Treatment>;
 
-	checkTreatmentTimeConflict(followUpRecord, treatment, errors);
+	if (donorDataToValidateWith) {
+		// If there is no survival time information then we can't do our time validations.
+		// This is possible when there is an exception on survival time
+		checkDonorTimeConflict(donorDataToValidateWith, followUpRecord, errors);
+		checkTreatmentTimeConflict(followUpRecord, treatment, errors);
+	}
 
 	const followUpClinicalInfo = getExistingFollowUp(existentDonor, followUpRecord);
 	// adding new follow up to this donor ?
@@ -117,45 +184,3 @@ export const validate = async (
 	}
 	return { errors };
 };
-
-function checkTreatmentTimeConflict(
-	followUpRecord: DeepReadonly<SubmittedClinicalRecord>,
-	treatment: DeepReadonly<Treatment>,
-	errors: SubmissionValidationError[],
-) {
-	// A follow up may or may not be associated with treatment
-	if (treatment == undefined) return;
-
-	if (
-		followUpRecord.interval_of_followup &&
-		treatment.clinicalInfo &&
-		treatment.clinicalInfo.treatment_start_interval &&
-		followUpRecord.interval_of_followup <= treatment.clinicalInfo.treatment_start_interval
-	) {
-		errors.push(
-			utils.buildSubmissionError(
-				followUpRecord,
-				DataValidationErrors.FOLLOW_UP_CONFLICING_INTERVAL,
-				FollowupFieldsEnum.interval_of_followup,
-				[],
-			),
-		);
-	}
-}
-
-function getExistingFollowUp(
-	existingDonor: DeepReadonly<Donor>,
-	record: DeepReadonly<SubmittedClinicalRecord>,
-) {
-	if (existingDonor.followUps) {
-		return getClinicalEntitiesFromDonorBySchemaName(
-			existingDonor,
-			ClinicalEntitySchemaNames.FOLLOW_UP,
-		).find(
-			(ci) =>
-				ci[FollowupFieldsEnum.submitter_follow_up_id] ==
-				record[FollowupFieldsEnum.submitter_follow_up_id],
-		);
-	}
-	return undefined;
-}
diff --git a/src/submission/validation-clinical/specimen.ts b/src/submission/validation-clinical/specimen.ts
@@ -29,6 +29,7 @@ import {
 	ClinicalEntitySchemaNames,
 	SpecimenFieldsEnum,
 	PrimaryDiagnosisFieldsEnum,
+	DonorFieldsEnum,
 } from '../../common-model/entities';
 import { DeepReadonly } from 'deep-freeze';
 import { Donor, PrimaryDiagnosis, Specimen } from '../../clinical/clinical-entities';
@@ -213,15 +214,15 @@ export const validate = async (
 	// validate allowed/unallowed fields
 	checkRequiredFields(specimen, specimenRecord, mergedDonor, errors);
 
-	// validate time conflict if needed
-	const donorDataToValidateWith = utils.getDataFromDonorRecordOrDonor(
+	const donorDataToValidateWith = utils.getSurvivalDataFromDonor(
 		specimenRecord,
 		mergedDonor,
-		errors,
 		SpecimenFieldsEnum.specimen_acquisition_interval,
 	);
 
 	if (donorDataToValidateWith) {
+		// If there is no survival time information then we can't do our time validations.
+		// This is possible when there is an exception on survival times
 		checkTimeConflictWithDonor(donorDataToValidateWith, specimenRecord, errors);
 	}
 

diff --git a/src/submission/validation-clinical/treatment.ts b/src/submission/validation-clinical/treatment.ts
@@ -25,6 +25,7 @@ import {
 	ClinicalTherapySchemaNames,
 	ClinicalTherapyType,
 	ClinicalUniqueIdentifier,
+	DonorFieldsEnum,
 	TreatmentFieldsEnum,
 } from '../../common-model/entities';
 import {
@@ -79,18 +80,16 @@ export const validate = async (
 		true,
 	);
 
-	// treatment.treatment_start_interval must be smaller than donor.survival_time
-	if (treatmentRecord[TreatmentFieldsEnum.treatment_start_interval]) {
-		const donorDataToValidateWith = utils.getDataFromDonorRecordOrDonor(
-			treatmentRecord,
-			mergedDonor,
-			errors,
-			TreatmentFieldsEnum.treatment_start_interval,
-		);
+	const donorDataToValidateWith = utils.getSurvivalDataFromDonor(
+		treatmentRecord,
+		mergedDonor,
+		TreatmentFieldsEnum.treatment_start_interval,
+	);
 
-		if (donorDataToValidateWith) {
-			checkDonorTimeConflict(donorDataToValidateWith, treatmentRecord, errors);
-		}
+	if (donorDataToValidateWith) {
+		// If there is no survival time information then we can't do our time validations.
+		// This is possible when there is an exception on survival time
+		checkDonorTimeConflict(donorDataToValidateWith, treatmentRecord, errors);
 	}
 
 	// Find existing follow ups for this donor