Skip to content

Commit

Permalink
🧪 Feature/ #1204 Fix Integration Tests (#1216)
Browse files Browse the repository at this point in the history 
* 🧹 Chore/ #1204 Mongo PR Cleanup (#1209)

* Npm audit fix

* Fix build issue

* Update Express

* Nodemailer

* Node-Vault

* jsonwebtoken

* Nodemon

* Multer

* @apollo/server

* Ego Utils

* Redo Npm audit fix

* Test spec files

* Bootstrap Import Organize

* donor-repo import organize

* Schemas File

* Move API Definition

* Registration Import organize

* Submission API cleanup

* Utils Imports

* Clinical.spec imports

* Schema Manager.spec imports

* Additional organize imports

* Unused CountBy Function

* Schema Imports

* Chore / Db Url Variable casing (#1212)

* Variable name changes

* Remove some test logging

* 🐬 Chore / Update MySql Integration Tests (#1213)

* Add dependencies

* Bootstrap / init / config changes

* Update service and server

* Fist 2 test file updates

* Further Test file updates

* TestUtils changes

* Last two test files

* Unused mongo container promise

* Additional Mongo syntax

* Dedupe RXNorm vars

* Move test vars to constants file

* Missing awaits + connection syntax update

* Remove try/catch

* 🦝 Chore/ Fix Mongo Test Containers + Connections (#1215)

* Fix connections - All tests passing

* Fix 'ns not found' dropCollection errors

* Reorg Donor Repo + Schemas

* Unused import

* Require RxNorm config values

* PR feedback pt 1

* PR feedback pt2

* Last RxNorm cleanup

* Mention recursive check in comment
  • Loading branch information
@demariadaniel
demariadaniel authored Nov 14, 2024
1 parent cd56ec3 commit 714dc1c
Show file tree
Hide file tree
Showing 22 changed files with 2,345 additions and 1,011 deletions.
2,187 changes: 1,751 additions & 436 deletions package-lock.json
View file

Large diffs are not rendered by default.

9 changes: 5 additions & 4 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"serve": "node dist/src/server.js",
"watch-node": "nodemon dist/src/server.js",
"test": "npm run unit-test && npm run int-test",
"int-test": "npm run build && DEBUG=testcontainers mocha --exit --timeout 60000 -r ts-node/register test/integration/**/*.ts",
"int-test": "npm run build && DEBUG=testcontainers mocha --exit --timeout 60000 -r ts-node/register test/integration/**/*.spec.ts",
"unit-test": "npm run build && mocha --exit --timeout 5000 -r ts-node/register test/unit/**/*.spec.ts test/unit/**/**/*.spec.ts",
"perf-test": "npm run build && mocha --exit --timeout 60000 -r ts-node/register test/performance-test/**/*.spec.ts",
"build-ts": "tsc",
Expand Down Expand Up @@ -40,6 +40,8 @@
"devDependencies": {
"@swc/core": "^1.3.32",
"@swc/helpers": "^0.4.14",
"@testcontainers/mongodb": "^10.14.0",
"@testcontainers/mysql": "^10.13.2",
"@types/adm-zip": "^0.4.33",
"@types/async": "^3.0.0",
"@types/bcrypt-nodejs": "^0.0.30",
Expand All @@ -65,7 +67,6 @@
"@types/mongoose-sequence": "^3.0.3",
"@types/morgan": "^1.7.35",
"@types/multer": "^1.3.7",
"@types/mysql": "^2.15.9",
"@types/node": "^12.0.10",
"@types/node-fetch": "^2.5.0",
"@types/node-vault": "^0.9.0",
Expand All @@ -91,7 +92,7 @@
"regenerator-runtime": "^0.13.11",
"shelljs": "^0.8.5",
"sinon": "^7.3.2",
"testcontainers": "^1.3.1",
"testcontainers": "10.13",
"ts-node": "^10.9.1",
"tslint": "^5.20.1",
"typescript": "^5.0.0"
Expand Down Expand Up @@ -132,7 +133,7 @@
"morgan": "^1.10.0",
"mquery": ">=3.2.3",
"multer": "^1.4.5-lts.1",
"mysql": "^2.18.1",
"mysql2": "^3.11.3",
"node-fetch": "^2.6.7",
"node-vault": "^0.10.2",
"node-worker-threads-pool": "^1.2.2",
Expand Down
42 changes: 21 additions & 21 deletions src/bootstrap.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,16 @@
*/

import mongoose from 'mongoose';
import { loggerFor } from './logger';
import { AppConfig, initConfigs, JWT_TOKEN_PUBLIC_KEY, RxNormDbConfig } from './config';
import * as dictionaryManager from './dictionary/manager';
import * as utils from './utils';
import { Pool } from 'mysql2/promise';
import fetch from 'node-fetch';
import { setStatus, Status } from './app-health';
import { AppConfig, initConfigs, JWT_TOKEN_PUBLIC_KEY, RxNormDbConfig } from './config';
import * as dictionaryManager from './dictionary/manager';
import { loggerFor } from './logger';
import { initPool } from './rxnorm/pool';
import * as persistedConfig from './submission/persisted-config/service';
import * as submissionUpdatesMessenger from './submission/submission-updates-messenger';
import { initPool } from './rxnorm/pool';
import { promisify } from 'bluebird';
import { Pool } from 'mysql';
import * as utils from './utils';

const L = loggerFor(__filename);

Expand Down Expand Up @@ -135,30 +134,31 @@ const setJwtPublicKey = (keyUrl: string) => {
getKey(1);
};

const setupRxNormConnection = (conf: RxNormDbConfig) => {
if (!conf.host) return;
const setupRxNormConnection = (config: RxNormDbConfig) => {
if (!config.host) return;
const { database, host, password, user, port, connectTimeout } = config;
const pool = initPool({
database: conf.database,
host: conf.host,
password: conf.password,
user: conf.user,
port: conf.port,
timeout: conf.timeout,
database,
host,
password,
user,
port,
connectTimeout,
});
pool.on('connection', () => setStatus('rxNormDb', { status: Status.OK }));

// check for rxnorm connection every 5 minutes
pingRxNorm(pool);
setInterval(async () => {
await pingRxNorm(pool);
}, 5 * 60 * 1000);
};

async function pingRxNorm(pool: Pool) {
try {
const query = promisify(pool.query).bind(pool);
await query('select 1');
const connection = await pool.getConnection();
await connection.query('select 1');
pool.releaseConnection(connection);
setStatus('rxNormDb', { status: Status.OK });

// Recursively checks for rxnorm connection every 5 minutes
setTimeout(pingRxNorm, 5 * 60 * 1000);
} catch (err) {
L.error('cannot get connection to rxnorm', err);
setStatus('rxNormDb', { status: Status.ERROR });
Expand Down
205 changes: 9 additions & 196 deletions src/clinical/donor-repo.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,17 @@
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

import { Donor } from './clinical-entities';
import { ClinicalDataQuery, ClinicalDonorEntityQuery } from './types';
import { getRequiredDonorFieldsForEntityTypes } from '../common-model/functions';
import mongoose, { PaginateModel } from 'mongoose';
import mongoosePaginate from 'mongoose-paginate-v2';
import { DeepReadonly } from 'deep-freeze';
import mongoose, { PaginateModel } from 'mongoose';
import { getRequiredDonorFieldsForEntityTypes } from '../common-model/functions';
import { F, MongooseUtils, notEmpty } from '../utils';
import { Donor } from './clinical-entities';
import { DonorDocument, DonorSchema } from './schemas';
import { ClinicalDataQuery, ClinicalDonorEntityQuery } from './types';

export const SUBMITTER_ID = 'submitterId';
export const SPECIMEN_SUBMITTER_ID = 'specimen.submitterId';
export const SPECIMEN_SAMPLE_SUBMITTER_ID = 'specimen.sample.submitterId';
const AutoIncrement = require('mongoose-sequence')(mongoose);

export enum DONOR_DOCUMENT_FIELDS {
SUBMITTER_ID = 'submitterId',
Expand Down Expand Up @@ -69,6 +68,10 @@ export type FindPaginatedProgramFilter = {
donorId?: { $in: '' };
};

const DonorModel = mongoose.model<DonorDocument>('Donor', DonorSchema) as PaginateModel<
DonorDocument
>;

export interface DonorRepository {
findByClinicalEntitySubmitterIdAndProgramId(
filters: DeepReadonly<FindByProgramAndSubmitterFilter>,
Expand Down Expand Up @@ -111,7 +114,6 @@ export interface DonorRepository {
create(donor: DeepReadonly<Partial<Donor>>): Promise<DeepReadonly<Donor>>;
update(donor: DeepReadonly<Donor>): Promise<DeepReadonly<Donor>>;
updateAll(donors: DeepReadonly<Donor>[]): Promise<DeepReadonly<Donor>[]>;
countBy(filter: any): Promise<number>;
}

// Mongoose implementation of the DonorRepository
Expand All @@ -124,9 +126,6 @@ export const donorDao: DonorRepository = {
.collection('donors')
.findOneAndUpdate({ donorId: donor.donorId }, { $set: donor });
},
async countBy(filter: any) {
return await DonorModel.count(filter).exec();
},

async deleteByProgramId(programId: string): Promise<void> {
await DonorModel.deleteMany({
Expand Down Expand Up @@ -525,189 +524,3 @@ async function findByProgramIdOmitMongoDocId(

return F(result as Donor[]);
}

type DonorDocument = mongoose.Document & Donor;

const SampleSchema = new mongoose.Schema(
{
sampleId: { type: Number, index: true, unique: true },
sampleType: { type: String },
submitterId: { type: String, required: true },
},
{ _id: false },
);

const SpecimenSchema = new mongoose.Schema(
{
specimenId: { type: Number, index: true, unique: true },
specimenTissueSource: { type: String },
clinicalInfo: {},
tumourNormalDesignation: String,
specimenType: String,
submitterId: { type: String, required: true },
samples: [SampleSchema],
},
{ _id: false, minimize: false }, // minimize false is to avoid omitting clinicalInfo:{}
);

const TherapySchema = new mongoose.Schema(
{
clinicalInfo: {},
therapyType: { type: String, required: true },
},
{ _id: false },
);

const TreatmentSchema = new mongoose.Schema(
{
clinicalInfo: {},
treatmentId: { type: Number },
therapies: [TherapySchema],
},
{ _id: false },
);
TreatmentSchema.index({ treatmentId: 1 }, { unique: true, sparse: true });
const FollowUpSchema = new mongoose.Schema(
{
followUpId: { type: Number },
clinicalInfo: {},
},
{ _id: false },
);
FollowUpSchema.index({ followUpId: 1 }, { unique: true, sparse: true });
const PrimaryDiagnosisSchema = new mongoose.Schema(
{
primaryDiagnosisId: { type: Number },
clinicalInfo: {},
},
{ _id: false },
);

PrimaryDiagnosisSchema.index({ primaryDiagnosisId: 1 }, { unique: true, sparse: true });

const FamilyHistorySchema = new mongoose.Schema(
{
familyHistoryId: { type: Number },
clinicalInfo: {},
},
{ _id: false },
);

FamilyHistorySchema.index({ familyHistoryId: 1 }, { unique: true, sparse: true });

const ExposureSchema = new mongoose.Schema(
{
exposureId: { type: Number },
clinicalInfo: {},
},
{ _id: false },
);

ExposureSchema.index({ exposureId: 1 }, { unique: true, sparse: true });

const BiomarkerSchema = new mongoose.Schema(
{
biomarkerId: { type: Number },
clinicalInfo: {},
},
{ _id: false },
);

BiomarkerSchema.index({ biomarkerId: 1 }, { unique: true, sparse: true });

const ComorbiditySchema = new mongoose.Schema(
{
comorbidityId: { type: Number },
clinicalInfo: {},
},
{ _id: false },
);

ComorbiditySchema.index({ comorbidityId: 1 }, { unique: true, sparse: true });

const DonorSchema = new mongoose.Schema(
{
donorId: { type: Number, index: true, unique: true },
gender: { type: String, required: true },
submitterId: { type: String, required: true },
programId: { type: String, required: true },
specimens: [SpecimenSchema],
clinicalInfo: {},
primaryDiagnoses: [PrimaryDiagnosisSchema],
familyHistory: [FamilyHistorySchema],
comorbidity: [ComorbiditySchema],
followUps: [FollowUpSchema],
treatments: [TreatmentSchema],
exposure: [ExposureSchema],
biomarker: [BiomarkerSchema],
schemaMetadata: {},
completionStats: {},
},
{ timestamps: true, minimize: false }, // minimize false is to avoid omitting clinicalInfo:{}
);

DonorSchema.index({ submitterId: 1, programId: 1 }, { unique: true });
DonorSchema.index({ 'specimens.submitterId': 1, programId: 1 }, { unique: true });
DonorSchema.index({ 'specimens.samples.submitterId': 1, programId: 1 }, { unique: true });

/**
* These had to read from process env and not use the AppConfig
* because these are global mongoose variables, they can't be called
* multiple times, and that makes them hard to test because tests depend
* on resetting the config and bootstraping but global variables keep their state.
*/
DonorSchema.plugin(AutoIncrement, {
inc_field: 'donorId',
start_seq: process.env.DONOR_ID_SEED || 250000,
});

DonorSchema.plugin(mongoosePaginate);

SpecimenSchema.plugin(AutoIncrement, {
inc_field: 'specimenId',
start_seq: process.env.SPECIMEN_ID_SEED || 210000,
});

SampleSchema.plugin(AutoIncrement, {
inc_field: 'sampleId',
start_seq: process.env.SAMPLE_ID_SEED || 610000,
});

FollowUpSchema.plugin(AutoIncrement, {
inc_field: 'followUpId',
start_seq: 1,
});

PrimaryDiagnosisSchema.plugin(AutoIncrement, {
inc_field: 'primaryDiagnosisId',
start_seq: 1,
});

FamilyHistorySchema.plugin(AutoIncrement, {
inc_field: 'familyHistoryId',
start_seq: 1,
});

ExposureSchema.plugin(AutoIncrement, {
inc_field: 'exposureId',
start_seq: 1,
});

BiomarkerSchema.plugin(AutoIncrement, {
inc_field: 'biomarkerId',
start_seq: 1,
});

ComorbiditySchema.plugin(AutoIncrement, {
inc_field: 'comorbidityId',
start_seq: 1,
});

TreatmentSchema.plugin(AutoIncrement, {
inc_field: 'treatmentId',
start_seq: 1,
});

export let DonorModel = mongoose.model<DonorDocument>('Donor', DonorSchema) as PaginateModel<
DonorDocument
>;
Loading

0 comments on commit 714dc1c

Please sign in to comment.