diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b08bcdeaf..e2749339e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -12,7 +12,8 @@ "customizations": { "vscode": { "extensions": [ - "ms-python.python" + "ms-python.python", + "vscode-es6-string-html" ] } }, diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 035941d66..1dad5e527 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -36,48 +36,3 @@ jobs: pip install ruff - name: ruff --> Check code formatting run: ruff format --check . - - # Use pipreqs to check for missing dependencies - pipreqs-check: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.12" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-dev.txt - - - name: Run pipreqs - run: | - pipreqs --savepath pipreqs.txt . 2>&1 | tee pipreqs_output.log - if grep -q 'WARNING: Package .* does not exist or network problems' pipreqs_output.log; then - missing_packages=$(grep 'WARNING: Package .* does not exist or network problems' pipreqs_output.log | sed -E 's/.*Package "(.*)" does not exist.*/\1/') - echo "ERROR: Add unresolved packages to requirements. Missing package(s): $missing_packages. Example: ' @ git+https://github.com//.git'" - exit 1 - fi - - - name: Compare requirements - run: | - # Extract and sort package names - awk -F'(=|==|>|>=|<|<=| @ )' '{gsub("-", "_", $1); print $1}' requirements.txt | tr '[:upper:]' '[:lower:]' | sort -u > requirements.compare - awk -F'(=|==|>|>=|<|<=| @ )' '{gsub("-", "_", $1); print $1}' pipreqs.txt | tr '[:upper:]' '[:lower:]' | sort -u > pipreqs.compare - - # Compare package lists - missing_packages=$(comm -23 pipreqs.compare requirements.compare) - if [ -z "$missing_packages" ]; then - echo "All packages in pipreqs are listed in requirements" - exit 0 - else - echo "Some packages in pipreqs are not listed in requirements" - echo "" - echo "=== Missing packages ===" - echo "$missing_packages" - exit 1 - fi diff --git a/conda_requirements.yml b/conda_requirements.yml index d4b14013a..6fd2603ca 100644 --- a/conda_requirements.yml +++ b/conda_requirements.yml @@ -1,9 +1,13 @@ +channels: + - conda-forge dependencies: - python=3.12 - - 'conda-forge::pango>=1.42.0' - - 'conda-forge::pandas>=1.3.2' - - conda-forge::psycopg2 - - conda-forge::open-fonts - - conda-forge::xorg-libxrender - - conda-forge::xorg-libxext - - conda-forge::xorg-libxau + - pango>=1.42.0 + - pandas>=1.3.2 + - psycopg2 + - open-fonts + - xorg-libxrender + - xorg-libxext + - xorg-libxau + - pip: + - '-r requirements.txt' diff --git a/requirements.txt b/requirements.txt index d2f1cd9aa..9a489695f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,8 +14,6 @@ nose numpy>=1.12.1 oauth2>=1.9.0.post1 openpyxl -pandas -psycopg2 pycryptodome>=3.6.1 pyparsing>=2.2.0 python-dateutil>=2.7.5 diff --git a/requirements_dev.txt b/requirements_dev.txt index 8a7fce3ae..92c89deec 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,3 @@ ipdb -pipreqs ruff selenium diff --git a/run_dir/design/element_flowcell.html b/run_dir/design/element_flowcell.html index 4b8335dba..b3c0f8f47 100644 --- a/run_dir/design/element_flowcell.html +++ b/run_dir/design/element_flowcell.html @@ -1,8 +1,16 @@ {% extends 'base.html' %} {% block stuff %} -
-

Element BioSciences (AVITI) run

-
Under construction. This page should contain information about a single element biosciences run.
+
+
+{% end %} + +{% block js %} + + + + + {% end %} + diff --git a/run_dir/design/element_flowcells.html b/run_dir/design/element_flowcells.html index 1ac353e41..5d1e46645 100644 --- a/run_dir/design/element_flowcells.html +++ b/run_dir/design/element_flowcells.html @@ -8,8 +8,8 @@

+ NGI Run ID Start date - Run name Run type Side Cycles @@ -21,8 +21,8 @@

+ NGI Run ID Start date - Run name Run type Side Cycles @@ -35,9 +35,8 @@

{% for onefc in element_fcs %} - {{ onefc.key[:10] }} - - {{ onefc.value['RunName'] }} + + {{ onefc.key }} {% if onefc.value.get('Outcome') == 'OutcomeCompleted' %} {% elif onefc.value.get('Outcome') == 'ongoing' %} @@ -48,26 +47,27 @@

{% end %} + {{ onefc.key[:8] }} - {{ onefc.value["RunType"] }} + {{ onefc.value.get("RunType") }} - {{ onefc.value["Side"] }} + {{ onefc.value.get("Side") }} - {{ onefc.value["Cycles"] }} + {{ onefc.value.get("Cycles") }} - {{ onefc.value["ThroughputSelection"] }} + {{ onefc.value.get("ThroughputSelection") }} - {{ onefc.value["KitConfiguration"] }} + {{ onefc.value.get("KitConfiguration") }} - {{ onefc.value["ChemistryVersion"] }} + {{ onefc.value.get("ChemistryVersion") }} - {{ onefc.value["Outcome"] }} + {{ onefc.value.get("Outcome") }} {% end %} diff --git a/run_dir/design/pricing_preview.html b/run_dir/design/pricing_preview.html index 2e44966aa..f76805635 100644 --- a/run_dir/design/pricing_preview.html +++ b/run_dir/design/pricing_preview.html @@ -10,7 +10,6 @@ {% block js %} - diff --git a/run_dir/static/js/element_flowcell.js b/run_dir/static/js/element_flowcell.js new file mode 100644 index 000000000..f2d7060f5 --- /dev/null +++ b/run_dir/static/js/element_flowcell.js @@ -0,0 +1,1450 @@ + +const vElementApp = { + data() { + return { + ngi_run_id: "", + flowcell: {}, + flowcell_fetched: false, + } + }, + computed: { + // Data getters // + run_status() { + return this.getValue(this.flowcell, "run_status", "N/A"); + }, + instrument_generated_files() { + return this.getValue(this.flowcell, "instrument_generated_files", {}); + }, + aviti_run_stats() { + return this.getValue(this.instrument_generated_files, "AvitiRunStats.json", {}); + }, + run_parameters() { + return this.$root.getValue(this.instrument_generated_files, "RunParameters.json", {}); + }, + run_stats() { + return this.getValue(this.aviti_run_stats, "RunStats", {}); + }, + lane_stats() { + return this.getValue(this.aviti_run_stats, "LaneStats", {}); + }, + demultiplex_stats() { + return this.getValue( + this.getValue(this.flowcell, "Element", {}), + "Demultiplex_Stats", {} + ); + }, + index_assignment_demultiplex() { + return this.getValue(this.demultiplex_stats, "Index_Assignment", {}); + }, + index_assignment_pre_demultiplex() { + return this.getValue(this.run_stats, "IndexAssignment", {}); + }, + unassiged_sequences_demultiplex() { + return this.getValue(this.demultiplex_stats, "Unassigned_Sequences", {}); + }, + // Re-structuring data // + grouped_lane_stats_pre_demultiplex() { + /* + This function groups the lane stats by lane number. + */ + + const groupedByLane = {}; + this.lane_stats.forEach(lane => { + const lane_number = lane["Lane"]; + if (!groupedByLane[lane_number]) { + groupedByLane[lane_number] = {}; + } + groupedByLane[lane_number] = lane; + }); + + return groupedByLane; + }, + grouped_index_assignment_post_demultiplex() { + /* + Index assignment from demultiplex info as presented by TACA + This function groups the info by lane number. + */ + const groupedByLane = {}; + this.index_assignment_demultiplex.forEach(sample => { + const lane = sample["Lane"]; + if (!groupedByLane[lane]) { + groupedByLane[lane] = []; + } + groupedByLane[lane].push(sample); + }); + + return groupedByLane; + }, + project_ids_to_names() { + const project_ids_to_names = {}; + this.flowcell.projects.forEach(project => { + project_ids_to_names[project["project_id"]] = project["project_name"]; + }); + return project_ids_to_names; + }, + // Checks // + lane_ids_match() { + /* Lane stats from the instrument might not match the demultiplexed lane stats + in case the instrument run was not started with a correct manifest file. + */ + const keysPreDemultiplex = Object.keys(this.grouped_lane_stats_pre_demultiplex).sort(); + const keysPostDemultiplex = Object.keys(this.grouped_index_assignment_post_demultiplex).sort(); + + return keysPreDemultiplex.length === keysPostDemultiplex.length && + keysPreDemultiplex.every((key, index) => key === keysPostDemultiplex[index]); + }, + demultiplex_stats_available() { + if (this.index_assignment_demultiplex) { + return true; + } else { + return false; + } + } + }, + methods: { + // Data getters // + getFlowcell() { + axios.get("/api/v1/element_flowcell/" + this.ngi_run_id) + .then(response => { + this.flowcell = response.data; + this.flowcell_fetched = true; + }) + .catch(error => { + console.log(error); + }); + }, + getValue(obj, key, defaultValue = "N/A") { + /* A helper function to get a value from an object, with a default value if not found */ + if (obj === null || obj == undefined || obj === "N/A") { + return defaultValue; + } + return obj.hasOwnProperty(key) ? obj[key] : defaultValue; + }, + // Formatting // + barcode(sample) { + let barcode_str = ""; + if (sample.hasOwnProperty("I1") && sample["I1"] !== "") { + barcode_str += sample["I1"]; + } else { + barcode_str += "N/A" + } + + if (sample.hasOwnProperty("I2") && sample["I2"] !== ""){ + barcode_str += "+" + sample["I2"]; + } + return barcode_str; + }, + formatNumberBases(number) { + if (number === "N/A") { + return "N/A"; + } else if (number < 1000000) { + return number + " bp"; + } else if (number < 1000000000) { + return (number / 1000000).toFixed(2) + " Mbp"; + } else { + return (number / 1000000000).toFixed(2) + " Gbp"; + } + }, + formatNumberFloat(value, decimalPoints=2) { + if (value === "N/A") { + return "N/A"; + } + const number = parseFloat(value); + if (isNaN(number)) { + return "N/A"; + } + return number.toFixed(decimalPoints); + }, + formatNumberLarge(value) { + if (value === "N/A") { + return "N/A"; + } + return value.toString().replace(/\B(?=(\d{3})+(?!\d))/g, " "); + } + } +} + +const app = Vue.createApp(vElementApp); + + +app.component('v-element-flowcell', { + props: ['ngi_run_id'], + computed: { + flowcell() { + return this.$root.flowcell; + }, + start_time() { + const dateStr = this.$root.getValue(this.$root.run_parameters, "Date", null); + if (dateStr) { + const date = new Date(dateStr); + const date_string = date.toLocaleDateString(); + const [day, month, year] = date_string.split("/"); + const date_formatted = `${year}-${month}-${day}`; + return `${date_formatted} ${date.toLocaleTimeString()}`; + } else { + return "N/A"; + } + }, + flowcell_id() { + return this.$root.getValue(this.$root.run_parameters, "FlowcellID"); + }, + side() { + return this.$root.getValue(this.$root.run_parameters, "Side"); + }, + instrument_name() { + return this.$root.getValue(this.$root.run_parameters, "InstrumentName"); + }, + run_setup() { + return `${this.chemistry_version} ${this.kit_configuration}, ${this.throughput_selection}`; + }, + cycles() { + const cycles = this.$root.getValue(this.$root.run_parameters, "Cycles", {}); + if (cycles === "N/A") { + return "N/A"; + } + let parts = []; + if (cycles.hasOwnProperty("R1")) { + parts.push(`${cycles["R1"]}nt(R1)`); + } + if (cycles.hasOwnProperty("I1")) { + parts.push(`${cycles["I1"]}nt(I1)`); + } + if (cycles.hasOwnProperty("I2")) { + parts.push(`${cycles["I2"]}nt(I2)`); + } + if (cycles.hasOwnProperty("R2")) { + parts.push(`${cycles["R2"]}nt(R2)`); + } + return parts.join('-'); + }, + throughput_selection() { + return this.$root.getValue(this.$root.run_parameters, "ThroughputSelection", "N/A") + " Throughput"; + }, + kit_configuration() { + return this.$root.getValue(this.$root.run_parameters, "KitConfiguration"); + }, + preparation_workflow() { + return this.$root.getValue(this.$root.run_parameters, "PreparationWorkflow"); + }, + chemistry_version() { + return this.$root.getValue(this.$root.run_parameters, "ChemistryVersion"); + }, + run_status_img_class() { + if (this.$root.run_status === "sequencing") { + return 'fa-solid fa-cash-register' + } else if (this.$root.run_status === "demultiplexing") { + return 'fa-solid fa-bring-front' + } else if (this.$root.run_status === "transferring") { + return 'fa-solid fa-upload' + } else if (this.$root.run_status === "archived") { + return 'fa-solid fa-check' + } else { + return 'fa-solid fa-question' + } + } + }, + mounted() { + this.$root.ngi_run_id = this.ngi_run_id; + this.$root.getFlowcell(); + }, + template: /*html*/` +
+ + Loading... +
+
+

Element BioSciences (AVITI) run {{ flowcell["NGI_run_id"]}} {{this.$root.run_status}}

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NGI Run ID{{ flowcell["NGI_run_id"] }}
Start time{{ this.start_time }}
Flowcell ID{{ flowcell_id }}
Side{{ side }}
Instrument{{ instrument_name }}
Run setup{{ run_setup }}
Cycles{{ cycles }}
Projects: +
+ + +
+
+
+
+

Flowcell Statistics

+ +
+
+ +
+ +
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+ + ` +}); + +app.component('v-element-run-stats', { + computed: { + polony_count() { + return this.$root.getValue(this.$root.run_stats, "PolonyCount"); + }, + pf_count() { + return this.$root.getValue(this.$root.run_stats, "PFCount"); + }, + percent_pf() { + return this.$root.getValue(this.$root.run_stats, "PercentPF"); + }, + total_yield() { + return this.$root.getValue(this.$root.run_stats, "TotalYield"); + }, + percent_assigned_reads() { + return this.$root.getValue(this.$root.index_assignment_pre_demultiplex, "PercentAssignedReads"); + } + }, + template: ` + + + + + + + + + + + + + + + + + + + + + + + +
Total Yield + + {{ this.$root.formatNumberBases(this.total_yield) }} + +
Polony Count{{ this.$root.formatNumberLarge(polony_count) }}
PF Count{{ this.$root.formatNumberLarge(pf_count) }}
% PF{{ this.$root.formatNumberFloat(percent_pf) }}
% Assigned Reads{{ this.$root.formatNumberFloat(percent_assigned_reads) }}
+ ` +}); + +app.component('v-element-lane-stats', { + data() { + return { + show_phiX_details_data: {}, + items_to_show_data: {} + } + }, + computed: { + unassigned_lane_stats() { + const groupedByLane = {}; + + this.$root.unassiged_sequences_demultiplex.forEach(sample => { + const lane = sample["Lane"]; + if (!groupedByLane[lane]) { + groupedByLane[lane] = []; + } + groupedByLane[lane].push(sample); + }); + + return groupedByLane; + }, + phiX_lane_stats_combined() { + const groupedByLane = {}; + + this.$root.index_assignment_demultiplex.forEach((sample) => { + if (! this.is_not_phiX(sample)) { + const lane = sample["Lane"]; + if (!groupedByLane[lane]) { + groupedByLane[lane] = { + "Project": "Control", + "SampleName": "PhiX", + "NumPoloniesAssigned": 0, + "PercentPoloniesAssigned": 0, + "Yield(Gb)": 0, + "Lane": lane, + "sub_demux_count": new Set(), + "PercentMismatch": [], + "PercentQ30": [], + "PercentQ40": [], + "QualityScoreMean": [] + } + } + // Summable values + groupedByLane[lane]["NumPoloniesAssigned"] += parseFloat(sample["NumPoloniesAssigned"]); + groupedByLane[lane]["PercentPoloniesAssigned"] += parseFloat(sample["PercentPoloniesAssigned"]); + groupedByLane[lane]["Yield(Gb)"] += parseFloat(sample["Yield(Gb)"]); + // List unique values + groupedByLane[lane]["sub_demux_count"].add(sample["sub_demux_count"]); + // Prepare for mean value-calculations + groupedByLane[lane]["PercentMismatch"].push(sample["PercentMismatch"] * sample["NumPoloniesAssigned"]); + groupedByLane[lane]["PercentQ30"].push(sample["PercentQ30"] * sample["NumPoloniesAssigned"]); + groupedByLane[lane]["PercentQ40"].push(sample["PercentQ40"] * sample["NumPoloniesAssigned"]); + groupedByLane[lane]["QualityScoreMean"].push(sample["QualityScoreMean"] * sample["NumPoloniesAssigned"]); + } + }) + + + // Calculate mean values + Object.entries(groupedByLane).forEach(([laneKey, lane]) => { + lane["PercentMismatch"] = lane["PercentMismatch"].reduce((a, b) => a + b, 0) / lane["NumPoloniesAssigned"]; + lane["PercentQ30"] = lane["PercentQ30"].reduce((a, b) => a + b, 0) / lane["NumPoloniesAssigned"]; + lane["PercentQ40"] = lane["PercentQ40"].reduce((a, b) => a + b, 0) / lane["NumPoloniesAssigned"]; + lane["QualityScoreMean"] = lane["QualityScoreMean"].reduce((a, b) => a + b, 0) / lane["NumPoloniesAssigned"]; + }) + + return groupedByLane; + }, + unassigned_lane_stats_combined() { + const samplesGroupedByLane = {}; + Object.entries(this.$root.grouped_index_assignment_post_demultiplex).forEach(entry => { + [lane_nr, samples] = entry; + samples.forEach(sample => { + if (!samplesGroupedByLane[lane_nr]) { + samplesGroupedByLane[lane_nr] = { + "NumPoloniesAssigned": 0, + "PercentPoloniesAssigned": 0, + "Lane": lane_nr, + } + + } + samplesGroupedByLane[lane_nr]["PercentPoloniesAssigned"] += parseFloat(sample["PercentPoloniesAssigned"]); + samplesGroupedByLane[lane_nr]["NumPoloniesAssigned"] += parseFloat(sample["NumPoloniesAssigned"]); + }); + }); + const groupedByLane = {}; + Object.values(samplesGroupedByLane).forEach(lane_summary => { + const lane = lane_summary["Lane"]; + groupedByLane[lane] = { + "Project": "Unassigned", + "SampleName": "Unassigned", + "NumPoloniesAssigned": '', // Need to fetch total from lane stats + "PercentPoloniesAssigned": 100 - lane_summary["PercentPoloniesAssigned"], + "Yield(Gb)": "", + "Lane": lane, + "sub_demux_count": "", + "PercentMismatch": "", + "PercentQ30": "", + "PercentQ40": "", + "QualityScoreMean": "" + } + }); + + return groupedByLane; + } + }, + methods: { + is_not_phiX(sample) { + return sample["SampleName"].indexOf("PhiX") === -1; + }, + items_to_show(laneKey) { + if (Object.keys(this.items_to_show_data).includes(laneKey)) { + return this.items_to_show_data[laneKey]; + } else { + return 5; + } + }, + show_more_items(laneKey) { + if (Object.keys(this.items_to_show_data).includes(laneKey)) { + this.items_to_show_data[laneKey] += 5; + } else { + this.items_to_show_data[laneKey] = 10; + } + }, + toggle_phiX_details(laneKey) { + if (Object.keys(this.show_phiX_details_data).includes(laneKey)) { + this.show_phiX_details_data[laneKey] = !this.show_phiX_details_data[laneKey]; + } else { + this.show_phiX_details_data[laneKey] = true; + } + }, + show_phiX_details(laneKey) { + if (Object.keys(this.show_phiX_details_data).includes(laneKey)) { + return this.show_phiX_details_data[laneKey]; + } else { + return false; + } + } + }, + template: /*html*/` + +

No demultiplex stats available

+ + + + + +
+

+ Lane {{ laneKey }} +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
Project NameSample NameYield (Gb)Num Polonies Assigned% Q30% Q40Barcode(s)% Assigned Reads% Assigned With MismatchesSub Demux CountQuality Score Mean
+ +
+ + + + +
+
+
+
+
+ + + + + + + + + + + + + + + + + + +
Sequence% UnassignedCount
{{ this.$root.barcode(unassigned_item)}}{{ this.$root.formatNumberFloat( this.$root.getValue(unassigned_item, "% Unassigned"), decimalPoints=5)}} {{ unassigned_item["Count"] }}
+ +
+
+
+
+
+
+ + ` +}); + +app.component('v-lane-stats-row', { + props: ['sample'], + computed: { + project_name() { + return this.$root.getValue(this.sample, "Project", "N/A").replace(/__/g, '.'); + }, + sub_demux_count() { + if (this.sample["sub_demux_count"] instanceof Set) { + return Array.from(this.sample["sub_demux_count"]).join(", "); + } else { + return this.sample["sub_demux_count"]; + } + } + }, + template: /*html*/` + + {{this.project_name}} + {{ sample["SampleName"] }} + {{ this.$root.formatNumberFloat(sample["Yield(Gb)"]) }} + {{ this.$root.formatNumberLarge(sample["NumPoloniesAssigned"]) }} + {{ this.$root.formatNumberFloat(sample["PercentQ30"]) }} + {{ this.$root.formatNumberFloat(sample["PercentQ40"]) }} + {{ this.$root.barcode(sample) }} + {{ this.$root.formatNumberFloat(sample["PercentPoloniesAssigned"]) }} + {{ this.$root.formatNumberFloat(sample["PercentMismatch"]) }} + {{ this.sub_demux_count }} + {{ this.$root.formatNumberFloat(sample["QualityScoreMean"]) }} + + ` +}); + +app.component('v-element-lane-stats-pre-demultiplex', { + computed: { + lanes() { + return this.$root.getValue(this.lane_stats, "Lanes", []); + } + }, + methods: { + index_assignments(lane) { + return this.$root.getValue(lane, "IndexAssignment", {}); + }, + index_samples(lane) { + return this.$root.getValue(this.index_assignments(lane), "IndexSamples", {}); + }, + project_name(sample) { + const project_id = sample["SampleName"].split("_")[0]; + if (!this.$root.project_ids_to_names[project_id]) { + return project_id; + } else { + return this.$root.project_ids_to_names[project_id].replace(/__/g, '.'); + } + }, + unassigned_sequences(lane) { + return this.$root.getValue(this.index_assignments(lane), "UnassignedSequences", {}); + }, + unassigned_sequences_percentage(lane) { + // Since not all unassigned sequences are reported, we calculate the percentage based on the assigned reads + return this.$root.formatNumberFloat( + 100 - this.$root.getValue(this.index_assignments(lane), "PercentAssignedReads", 0) + ) + } + }, + template: /*html*/` +
+

+ Lane {{ lane["Lane"] }} + Pre-demultiplexing +

+ + + +

Index Assignments

+ + + + + + + + + + + + + + + + + + + + + + + +
Project NameSample Name% Assigned Reads% Assigned With Mismatches
{{ this.project_name(sample) }}{{ sample["SampleName"] }}{{ this.$root.formatNumberFloat(sample["PercentAssignedReads"]) }}{{ this.$root.formatNumberFloat(sample["PercentMismatch"]) }}
Undetermined{{this.unassigned_sequences_percentage(lane)}}
+ + + +
+
+
+
+ + + + + + + + + + + + + +
Sequence% Occurence
{{ unassigned_item["I1"] }}+{{ unassigned_item["I2"]}}{{ this.$root.formatNumberFloat(unassigned_item["PercentOcurrence"], decimalPoints=5) }}
+
+
+
+
+ ` + +}); + +app.component('v-element-lane-summary', { + props: ['lane'], + methods: { + total_lane_yield(lane) { + return this.$root.formatNumberLarge(this.$root.getValue(lane, "TotalYield")); + }, + total_lane_yield_formatted(lane) { + return this.$root.formatNumberBases(this.$root.getValue(lane, "TotalYield")); + }, + percent_assigned_reads(lane) { + return this.$root.formatNumberFloat( + this.$root.getValue( + this.$root.getValue(lane, "IndexAssignment", {}), + "PercentAssignedReads" + ) + ) + }, + polony_count(lane) { + return this.$root.formatNumberLarge(this.$root.getValue(lane, "PolonyCount")); + }, + pf_count(lane) { + return this.$root.formatNumberLarge(this.$root.getValue(lane, "PFCount")); + }, + percent_pf(lane) { + return this.$root.formatNumberFloat(this.$root.getValue(lane, "PercentPF")); + } + }, + template: /*html*/` + + + + + + + + + + + + + + + + + + + +
Total Yield + + {{ total_lane_yield_formatted(lane) }} + + Polony Count{{ polony_count(lane) }}PF Count{{ pf_count(lane) }}% PF{{ percent_pf(lane) }}% Assigned Reads{{ percent_assigned_reads(lane) }}
+

Lane summary statistics is from pre-demultiplexing sources

+ + ` +}) + +app.component('v-element-project-yields', { + methods: { + project_stats(laneKey) { + const groupedByProject = {}; + this.$root.index_assignment_demultiplex.forEach(sample => { + const project = sample["Project"]; + if (laneKey == sample["Lane"]) { + + if (!groupedByProject[project]) { + groupedByProject[project] = { + "Project": project, + "TotalYield": 0, + "NumPoloniesAssigned": 0, + "PercentQ30": 0, + "PercentQ40": 0 + } + } + groupedByProject[project]["TotalYield"] += parseFloat(sample["Yield(Gb)"]); + groupedByProject[project]["NumPoloniesAssigned"] += parseFloat(sample["NumPoloniesAssigned"]); + groupedByProject[project]["PercentQ30"] += parseFloat(sample["PercentQ30"] * sample["NumPoloniesAssigned"]); + groupedByProject[project]["PercentQ40"] += parseFloat(sample["PercentQ40"] * sample["NumPoloniesAssigned"]); + } + }); + + Object.entries(groupedByProject).forEach(([projectKey, project]) => { + project["PercentQ30"] /= project["NumPoloniesAssigned"]; + project["PercentQ40"] /= project["NumPoloniesAssigned"]; + }) + + return groupedByProject; + }, + fraction_of_lane(project, laneKey) { + if (this.$root.lane_ids_match) { + let lane_count = this.$root.getValue(this.$root.grouped_lane_stats_pre_demultiplex[laneKey], "PFCount", 0) + if (lane_count === 0) { + return 'N/A' + } else { + return this.$root.formatNumberFloat(project['NumPoloniesAssigned'] / lane_count * 100); + } + } else { + return 'N/A' + } + } + }, + template: /*html*/` + +

No demultiplex stats available

+ + + + + +
+

Lane {{ laneKey }}

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Project NameTotal Yield (Gb)Total Num Polonies AssignedMean % Q30Mean % Q40Obtained Lane Yield %
{{ project_name.replace(/__/g, '.') }}{{ this.$root.formatNumberFloat(project["TotalYield"]) }}{{ this.$root.formatNumberLarge(project["NumPoloniesAssigned"]) }}{{ this.$root.formatNumberFloat(project["PercentQ30"]) }}{{ this.$root.formatNumberFloat(project["PercentQ40"]) }}{{ fraction_of_lane(project, laneKey) }}
+
+ + ` +}); + +app.component('v-element-tooltip', { + props: ['title'], + mounted() { + this.$nextTick(function() { + this.tooltip = new bootstrap.Tooltip(this.$el) + }) + }, + template: ` + + + + ` +}); + +app.component('v-element-graphs', { + data() { + return { + include_R1: true, + include_R2: true, + graph_warnings: [], + filter_first_cycles: 1, + filter_last_cycles: 1, + use_dynamic_yscale: true, + } + }, + computed: { + flowcell() { + return this.$root.flowcell; + }, + reads() { + return this.$root.getValue(this.$root.run_stats, "Reads", []); + }, + R1_read_cycles() { + let filtered_values = this.reads.filter(read => read['Read'] == 'R1') + if (filtered_values.length === 0) { + return []; + } + + return this.$root.getValue(filtered_values[0], 'Cycles', []) + }, + R2_read_cycles() { + let filtered_values = this.reads.filter(read => read['Read'] == 'R2') + if (filtered_values.length === 0) { + return []; + } + + return this.$root.getValue(filtered_values[0], 'Cycles', []) + }, + categories() { + // Check if R1 is in end_filter + let categories_R1 = this.R1_read_cycles.map(cycle => `Cycle ${cycle.Cycle}`); + + let categories_R2 = this.R2_read_cycles.map(cycle => `Cycle ${cycle.Cycle}`); + + if (categories_R1.length !== categories_R2.length) { + console.log("The lengths of categories_R1 and categories_R2 are different."); + console.log("Length of categories_R1:", categories_R1.length); + console.log("Length of categories_R2:", categories_R2.length); + } + let categories_differ = false; + for (let i = 0; i < Math.max(categories_R1.length, categories_R2.length); i++) { + if (categories_R1[i] !== categories_R2[i]) { + categories_differ = true; + console.log(`Difference found at index ${i}:`); + console.log(`categories_R1[${i}]:`, categories_R1[i]); + console.log(`categories_R2[${i}]:`, categories_R2[i]); + } + } + + if (categories_differ) { + this.graph_warnings.push("Warning! R1 and R2 x-axis are not identical, using R1 axis"); + } + + return categories_R1; + }, + }, + methods: { + empirical_quality_score(percent_error_rate) { + return -10 * Math.log10(percent_error_rate/100); + }, + summary_graph() { + if (this.categories.length === 0) { + return; + } + /* Filter the first categories */ + let filtered_categories = this.categories.slice(this.filter_first_cycles); + + /* filter the last categories */ + /* The last value seems to be weird for quality */ + if (this.filter_last_cycles > 0) { + filtered_categories = filtered_categories.slice(0, -this.filter_last_cycles); + } + + let R1_percentQ30 = []; + let R1_percentQ40 = []; + let R1_averageQScore = []; + let R1_phiX_error_rate = []; + let R1_phiX_empirical_error_rate = []; + let R1_base_composition = {}; + + let series = []; + let avg_series = []; + let phiX_error_rate_series = []; + let phiX_empirical_error_rate_series = []; + let R1_base_composition_series = []; + let R2_base_composition_series = []; + + if (this.include_R1) { + R1_percentQ30 = this.R1_read_cycles.map(cycle => cycle.PercentQ30); + R1_percentQ40 = this.R1_read_cycles.map(cycle => cycle.PercentQ40); + R1_averageQScore = this.R1_read_cycles.map(cycle => cycle.AverageQScore); + R1_phiX_error_rate = this.R1_read_cycles.map(cycle => cycle.PercentPhixErrorRate); + R1_phiX_empirical_error_rate = R1_phiX_error_rate.map(error_rate => this.empirical_quality_score(error_rate)); + R1_base_composition['A'] = this.R1_read_cycles.map(cycle => cycle.BaseComposition['A']); + R1_base_composition['C'] = this.R1_read_cycles.map(cycle => cycle.BaseComposition['C']); + R1_base_composition['G'] = this.R1_read_cycles.map(cycle => cycle.BaseComposition['G']); + R1_base_composition['T'] = this.R1_read_cycles.map(cycle => cycle.BaseComposition['T']); + + /* Filter the first values */ + R1_percentQ30 = R1_percentQ30.slice(this.filter_first_cycles); + R1_percentQ40 = R1_percentQ40.slice(this.filter_first_cycles); + R1_averageQScore = R1_averageQScore.slice(this.filter_first_cycles); + R1_phiX_error_rate = R1_phiX_error_rate.slice(this.filter_first_cycles); + R1_phiX_empirical_error_rate = R1_phiX_empirical_error_rate.slice(this.filter_first_cycles); + R1_base_composition['A'] = R1_base_composition['A'].slice(this.filter_first_cycles); + R1_base_composition['C'] = this.R1_read_cycles.map(cycle => cycle.BaseComposition['C']).slice(this.filter_first_cycles); + R1_base_composition['G'] = this.R1_read_cycles.map(cycle => cycle.BaseComposition['G']).slice(this.filter_first_cycles); + R1_base_composition['T'] = this.R1_read_cycles.map(cycle => cycle.BaseComposition['T']).slice(this.filter_first_cycles); + + /* Filter the last values */ + if (this.filter_last_cycles > 0) { + R1_percentQ30 = R1_percentQ30.slice(0, -this.filter_last_cycles); + R1_percentQ40 = R1_percentQ40.slice(0, -this.filter_last_cycles); + R1_averageQScore = R1_averageQScore.slice(0, -this.filter_last_cycles); + R1_phiX_error_rate = R1_phiX_error_rate.slice(0, -this.filter_last_cycles); + R1_phiX_empirical_error_rate = R1_phiX_empirical_error_rate.slice(0, -this.filter_last_cycles); + R1_base_composition['A'] = R1_base_composition['A'].slice(0, -this.filter_last_cycles); + R1_base_composition['C'] = this.R1_read_cycles.map(cycle => cycle.BaseComposition['C']).slice(0, -this.filter_last_cycles); + R1_base_composition['G'] = this.R1_read_cycles.map(cycle => cycle.BaseComposition['G']).slice(0, -this.filter_last_cycles); + R1_base_composition['T'] = this.R1_read_cycles.map(cycle => cycle.BaseComposition['T']).slice(0, -this.filter_last_cycles); + } + + series.push({ + name: 'R1 Percent Q30', + data: R1_percentQ30, + }) + + series.push( + { + name: 'R1 Percent Q40', + data: R1_percentQ40 + }) + + avg_series.push( + { + name: 'R1 Average Q Score', + data: R1_averageQScore + }) + + phiX_error_rate_series.push( + { + name: 'R1 Percent PhiX Error Rate', + data: R1_phiX_error_rate + }) + + phiX_empirical_error_rate_series.push( + { + name: 'R1 PhiX Empirical Quality Score', + data: R1_phiX_empirical_error_rate + }) + + R1_base_composition_series.push( + { + name: 'R1 Base Composition A', + data: this.R1_read_cycles.map(cycle => cycle.BaseComposition['A']) + }) + R1_base_composition_series.push( + { + name: 'R1 Base Composition C', + data: this.R1_read_cycles.map(cycle => cycle.BaseComposition['C']) + }) + R1_base_composition_series.push( + { + name: 'R1 Base Composition G', + data: this.R1_read_cycles.map(cycle => cycle.BaseComposition['G']) + }) + R1_base_composition_series.push( + { + name: 'R1 Base Composition T', + data: this.R1_read_cycles.map(cycle => cycle.BaseComposition['T']) + }) + } + + let R2_percentQ30 = []; + let R2_percentQ40 = []; + let R2_averageQScore = []; + let R2_phiX_error_rate = []; + let R2_phiX_empirical_error_rate = []; + let R2_base_composition = {}; + + if (this.include_R2) { + R2_percentQ30 = this.R2_read_cycles.map(cycle => cycle.PercentQ30); + R2_percentQ40 = this.R2_read_cycles.map(cycle => cycle.PercentQ40); + R2_averageQScore = this.R2_read_cycles.map(cycle => cycle.AverageQScore); + R2_phiX_error_rate = this.R2_read_cycles.map(cycle => cycle.PercentPhixErrorRate); + R2_phiX_empirical_error_rate = R2_phiX_error_rate.map(error_rate => this.empirical_quality_score(error_rate)); + R2_base_composition['A'] = this.R2_read_cycles.map(cycle => cycle.BaseComposition['A']); + R2_base_composition['C'] = this.R2_read_cycles.map(cycle => cycle.BaseComposition['C']); + R2_base_composition['G'] = this.R2_read_cycles.map(cycle => cycle.BaseComposition['G']); + R2_base_composition['T'] = this.R2_read_cycles.map(cycle => cycle.BaseComposition['T']); + + /* Filter the first values */ + R2_percentQ30 = R2_percentQ30.slice(this.filter_first_cycles); + R2_percentQ40 = R2_percentQ40.slice(this.filter_first_cycles); + R2_averageQScore = R2_averageQScore.slice(this.filter_first_cycles); + R2_phiX_error_rate = R2_phiX_error_rate.slice(this.filter_first_cycles); + R2_phiX_empirical_error_rate = R2_phiX_empirical_error_rate.slice(this.filter_first_cycles); + R2_base_composition['A'] = R2_base_composition['A'].slice(this.filter_first_cycles); + R2_base_composition['C'] = this.R2_read_cycles.map(cycle => cycle.BaseComposition['C']).slice(this.filter_first_cycles); + R2_base_composition['G'] = this.R2_read_cycles.map(cycle => cycle.BaseComposition['G']).slice(this.filter_first_cycles); + R2_base_composition['T'] = this.R2_read_cycles.map(cycle => cycle.BaseComposition['T']).slice(this.filter_first_cycles); + + /* Filter the last values */ + if (this.filter_last_cycles > 0) { + R2_percentQ30 = R2_percentQ30.slice(0, -this.filter_last_cycles); + R2_percentQ40 = R2_percentQ40.slice(0, -this.filter_last_cycles); + R2_averageQScore = R2_averageQScore.slice(0, -this.filter_last_cycles); + R2_phiX_error_rate = R2_phiX_error_rate.slice(0, -this.filter_last_cycles); + R2_phiX_empirical_error_rate = R2_phiX_empirical_error_rate.slice(0, -this.filter_last_cycles); + R2_base_composition['A'] = R2_base_composition['A'].slice(0, -this.filter_last_cycles); + R2_base_composition['C'] = this.R2_read_cycles.map(cycle => cycle.BaseComposition['C']).slice(0, -this.filter_last_cycles); + R2_base_composition['G'] = this.R2_read_cycles.map(cycle => cycle.BaseComposition['G']).slice(0, -this.filter_last_cycles); + R2_base_composition['T'] = this.R2_read_cycles.map(cycle => cycle.BaseComposition['T']).slice(0, -this.filter_last_cycles); + } + + series.push({ + name: 'R2 Percent Q30', + data: R2_percentQ30, + dashStyle: 'Dash' // Set dash style for R2 series + }); + + series.push({ + name: 'R2 Percent Q40', + data: R2_percentQ40, + dashStyle: 'Dash' // Set dash style for R2 series + }); + + avg_series.push({ + name: 'R2 Average Q Score', + data: R2_averageQScore, + dashStyle: 'Dash' // Set dash style for R2 series + }); + + phiX_error_rate_series.push({ + name: 'R2 Percent PhiX Error Rate', + data: R2_phiX_error_rate, + dashStyle: 'Dash' // Set dash style for R2 series + }); + + phiX_empirical_error_rate_series.push({ + name: 'R2 PhiX Empirical Quality Score', + data: R2_phiX_empirical_error_rate, + dashStyle: 'Dash' // Set dash style for R2 series + }); + + R2_base_composition_series.push({ + name: 'R2 Base Composition A', + data: R2_base_composition['A'], + dashStyle: 'Dash' // Set dash style for R2 series + }); + + R2_base_composition_series.push({ + name: 'R2 Base Composition C', + data: R2_base_composition['C'], + dashStyle: 'Dash' // Set dash style for R2 series + }); + + R2_base_composition_series.push({ + name: 'R2 Base Composition G', + data: R2_base_composition['G'], + dashStyle: 'Dash' // Set dash style for R2 series + }); + + R2_base_composition_series.push({ + name: 'R2 Base Composition T', + data: R2_base_composition['T'], + dashStyle: 'Dash' // Set dash style for R2 series + }); + } + + Highcharts.chart('SummaryPlotPercentQuality', { + chart: { + type: 'spline' + }, + title: { + text: '% Quality' + }, + xAxis: { + categories: filtered_categories + }, + yAxis: { + title: { + text: 'Percent Q30/Q40' + }, + min: this.use_dynamic_yscale ? null : 0, // Conditionally set the minimum value + max: this.use_dynamic_yscale ? null : 100 // Conditionally set the maximum value + }, + series: series.map(s => ({ + ...s, + marker: { + enabled: false, // Set to false to hide markers + } + })) + }) + + Highcharts.chart('SummaryPlotAvgQuality', { + chart: { + type: 'spline' + }, + title: { + text: 'Average Quality Score' + }, + xAxis: { + categories: filtered_categories + }, + yAxis: { + title: { + text: 'Average Q Score' + }, + min: this.use_dynamic_yscale ? null : 0, + max: this.use_dynamic_yscale ? null : 50, + }, + series: avg_series.map(s => ({ + ...s, + marker: { + enabled: false, // Set to false to hide markers + } + })) + }); + + Highcharts.chart('SummaryPlotPhiXErrorRate', { + chart: { + type: 'spline' + }, + title: { + text: '% PhiX Error Rate' + }, + xAxis: { + categories: filtered_categories + }, + yAxis: { + title: { + text: '% Error Rate' + }, + min: this.use_dynamic_yscale ? null : 0, + max: this.use_dynamic_yscale ? null : 20, + }, + series: phiX_error_rate_series.map(s => ({ + ...s, + marker: { + enabled: false, // Set to false to hide markers + } + })) + }); + + Highcharts.chart('SummaryPlotEmpiricalQuality', { + chart: { + type: 'spline' + }, + title: { + text: 'PhiX Empirical Quality Score' + }, + xAxis: { + categories: filtered_categories + }, + yAxis: { + title: { + text: 'Quality Score' + }, + min: this.use_dynamic_yscale ? null : 0, + max: this.use_dynamic_yscale ? null : 50, + }, + series: phiX_empirical_error_rate_series.map(s => ({ + ...s, + marker: { + enabled: false, // Set to false to hide markers + } + })) + }); + + Highcharts.chart('SummaryPlotBaseComposition_R1', { + chart: { + type: 'spline' + }, + title: { + text: 'R1 Base Composition' + }, + xAxis: { + categories: filtered_categories + }, + yAxis: { + title: { + text: 'Base Composition' + }, + min: this.use_dynamic_yscale ? null : 0, + max: this.use_dynamic_yscale ? null : 60, + }, + series: R1_base_composition_series.map(s => ({ + ...s, + marker: { + enabled: false, // Set to false to hide markers + } + })) + }); + + Highcharts.chart('SummaryPlotBaseComposition_R2', { + chart: { + type: 'spline' + }, + title: { + text: 'R2 Base Composition' + }, + xAxis: { + categories: filtered_categories + }, + yAxis: { + title: { + text: 'Base Composition' + }, + min: this.use_dynamic_yscale ? null : 0, + max: this.use_dynamic_yscale ? null : 60, + }, + series: R2_base_composition_series.map(s => ({ + ...s, + marker: { + enabled: false, // Set to false to hide markers + } + })) + }); + }, + }, + mounted() { + this.$nextTick(function() { + if (this.$root.flowcell_fetched) { + this.summary_graph(); + } + }); + }, + watch: { + include_R1: function() { + this.summary_graph(); + }, + include_R2: function() { + this.summary_graph(); + }, + filter_first_cycles: function() { + this.summary_graph(); + }, + filter_last_cycles: function() { + this.summary_graph(); + }, + use_dynamic_yscale: function() { + this.summary_graph(); + } + }, + template: /*html*/` +
+
+
+ + +
+
+ + +
+ +
+ + +
+ +
+
+ + + + + +
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ` +}); + +app.mount("#element_vue_app"); \ No newline at end of file diff --git a/status/flowcell.py b/status/flowcell.py index dfa734306..091bebdb3 100644 --- a/status/flowcell.py +++ b/status/flowcell.py @@ -463,13 +463,82 @@ def get(self, name): t.generate( gs_globals=self.application.gs_globals, user=self.get_current_user(), + ngi_run_id=name, ) ) +def get_project_ids_from_names(project_names: list, projects_db) -> list[dict]: + """Given a list of project names, perform a lookup to the projects db and return a json-style list of projects""" + projects = [] + for project_name in project_names: + rows = projects_db.view("projects/name_to_id")[project_name].rows + if rows: + projects.append( + { + "project_id": rows[0].value, + "project_name": project_name + } + ) + + return projects + + +def get_project_names_from_ids(project_ids: list, projects_db) -> list[dict]: + """Given a list of project ids, perform a lookup to the projects db and return a json-style list of projects""" + projects = [] + for project_id in project_ids: + rows = projects_db.view("projects/id_to_name")[project_id].rows + if rows: + projects.append( + { + "project_id": project_id, + "project_name": rows[0].value + } + ) + + return projects + + class ElementFlowcellDataHandler(SafeHandler): def get(self, name): - flowcell = self.application.element_runs_db.get(name) - self.write(flowcell) + rows = self.application.element_runs_db.view('info/id', include_docs=True)[name].rows + if rows: + flowcell = rows[0].doc + + # Collect all project names + project_names = [] + + demultiplexing_done = False + if flowcell.get('Element', {}).get('Demultiplex_Stats', {}).get('Index_Assignment'): + demultiplexing_done = True + + if demultiplexing_done: + samples_with_duplicates = [ + sample for sample in flowcell.get('Element', {}).get('Demultiplex_Stats', {}).get('Index_Assignment', []) + ] + project_names_with_duplicates = [sample.get('Project').replace('__', '.') for sample in samples_with_duplicates if sample.get('Project')] + project_names = list(set(project_names_with_duplicates)) + projects = get_project_ids_from_names(project_names, self.application.projects_db) + else: + project_ids = [] + for lane in flowcell.get('instrument_generated_files', {}).get('AvitiRunStats.json', {}).get('LaneStats', {}): + for sample in lane.get('IndexAssignments', {}).get('IndexSamples', {}): + sample_name = sample.get('SampleName') + # Check that the sample name is on the format PX..X_Y..Y" + if re.match(r"^P\d+_\d+$", sample_name): + # Parse out the PXXXXXX number from the sample name on the format " + project_id = sample_name.split('_')[0] + project_ids.append(project_id) + + project_ids = list(set(project_ids)) + projects = get_project_names_from_ids(project_ids, self.application.projects_db) + + flowcell['projects'] = projects + + self.write(flowcell) + else: + self.set_status(404) + self.write({"error": f"No element flowcell found for run ID {name}"}) class ONTFlowcellHandler(SafeHandler): """Serves a page which shows information for a given ONT flowcell.""" diff --git a/status_app.py b/status_app.py index db662281b..895fe6fc2 100644 --- a/status_app.py +++ b/status_app.py @@ -33,12 +33,7 @@ from status.controls import ControlsHandler from status.data_deliveries_plot import DataDeliveryHandler, DeliveryPlotHandler from status.deliveries import DeliveriesPageHandler -from status.flowcell import ( - ElementFlowcellHandler, - FlowcellHandler, - ONTFlowcellHandler, - ONTReportHandler, -) +from status.flowcell import FlowcellHandler, ElementFlowcellHandler, ElementFlowcellDataHandler, ONTFlowcellHandler, ONTReportHandler from status.flowcells import ( FlowcellDemultiplexHandler, FlowcellLinksDataHandler, @@ -237,6 +232,7 @@ def __init__(self, settings): ), ("/api/v1/draft_cost_calculator", PricingDraftDataHandler), ("/api/v1/draft_sample_requirements", SampleRequirementsDraftDataHandler), + ("/api/v1/element_flowcell/([^/]*$)", ElementFlowcellDataHandler), ("/api/v1/flowcells", FlowcellsDataHandler), ("/api/v1/flowcell_info2/([^/]*)$", FlowcellsInfoDataHandler), ("/api/v1/flowcell_info/([^/]*)$", OldFlowcellsInfoDataHandler),