Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EVA-3567 - Add basic labels to release accession #444

Merged
merged 3 commits into from
May 14, 2024
Merged

EVA-3567 - Add basic labels to release accession #444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9adcaed
basic labels
tcezard May 7, 2024
e9674f2
Add basic labels and fix one command
tcezard May 8, 2024
4e921f2
add tasks specific labels
tcezard May 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...ion-release-automation/run_release_in_embassy/copy_accessioning_collections_to_embassy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def remote_db_is_empty(local_forwarded_port, assembly_accession, destination_db_
logger.info(f"check if: {assembly_accession} has been copied")
command = f"mongosh --eval 'use {destination_db_name}' --eval 'db.stats()' --port {local_forwarded_port} " \
"| grep 'dataSize' | awk '{print substr($2,1,length($2)-1)}'"
output = run_command_with_output(command)
output = run_command_with_output('Check that data exists in mongodb', command, return_process_output=True)
return output.strip() == '0'


Expand Down
116 changes: 79 additions & 37 deletions eva-accession-release-automation/run_release_in_embassy/run_release_for_species.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,40 +31,70 @@
copy_process = "copy_accessioning_collections_to_embassy"
# Processes, in order, that make up the workflow and the arguments that they take
workflow_process_arguments_map = collections.OrderedDict(
[
("initiate_release_status_for_assembly", ["private-config-xml-file", "profile", "release-species-inventory-table",
"taxonomy-id", "assembly-accession", "release-version"]),
(copy_process, ["private-config-xml-file", "profile", "taxonomy-id", "assembly-accession",
"release-species-inventory-table", "release-version", "dump-dir"]),
("run_release_for_assembly", ["private-config-xml-file", "profile", "taxonomy-id",
"assembly-accession", "release-species-inventory-table",
"release-version", "species-release-folder", "release-jar-path"]),
("merge_dbsnp_eva_release_files", ["private-config-xml-file", "profile", "bgzip-path", "bcftools-path",
"vcf-sort-script-path", "taxonomy-id", "assembly-accession",
"release-species-inventory-table", "release-version",
"species-release-folder"]),
("sort_bgzip_index_release_files", ["bgzip-path", "bcftools-path",
"vcf-sort-script-path", "taxonomy-id", "assembly-accession",
"species-release-folder"]),
("validate_release_vcf_files", ["private-config-xml-file", "profile", "taxonomy-id",
"assembly-accession", "release-species-inventory-table", "release-version",
"species-release-folder",
"vcf-validator-path", "assembly-checker-path"]),
("analyze_vcf_validation_results", ["species-release-folder", "assembly-accession"]),
("count_rs_ids_in_release_files", ["count-ids-script-path", "taxonomy-id", "assembly-accession",
"species-release-folder"]),
("validate_rs_release_files", ["private-config-xml-file", "profile", "taxonomy-id", "assembly-accession",
"release-species-inventory-table", "release-version", "species-release-folder"]),
('update_sequence_names_to_ena', ["taxonomy-id", "assembly-accession", "species-release-folder",
'sequence-name-converter-path', 'bcftools-path']),
("update_release_status_for_assembly", ["private-config-xml-file", "profile", "release-species-inventory-table",
"taxonomy-id", "assembly-accession", "release-version"])
])
{
"initiate_release_status_for_assembly": (
["private-config-xml-file", "profile", "release-species-inventory-table", "taxonomy-id",
"assembly-accession", "release-version"],
'short_time', 'small_mem'
),
copy_process: (
["private-config-xml-file", "profile", "taxonomy-id", "assembly-accession",
"release-species-inventory-table", "release-version", "dump-dir"],
'long_time', 'small_mem'
),
"run_release_for_assembly":(
["private-config-xml-file", "profile", "taxonomy-id", "assembly-accession",
"release-species-inventory-table", "release-version", "species-release-folder", "release-jar-path"],
'long_time', 'med_mem'
),
"merge_dbsnp_eva_release_files": (
["private-config-xml-file", "profile", "bgzip-path", "bcftools-path", "vcf-sort-script-path",
"taxonomy-id", "assembly-accession", "release-species-inventory-table", "release-version",
"species-release-folder"],
'long_time', 'med_mem'
),
"sort_bgzip_index_release_files": (
["bgzip-path", "bcftools-path", "vcf-sort-script-path", "taxonomy-id", "assembly-accession",
"species-release-folder"],
'long_time', 'med_mem'
),
"validate_release_vcf_files": (
["private-config-xml-file", "profile", "taxonomy-id", "assembly-accession",
"release-species-inventory-table", "release-version", "species-release-folder",
"vcf-validator-path", "assembly-checker-path"],
'long_time', 'med_mem'
),
"analyze_vcf_validation_results": (
["species-release-folder", "assembly-accession"],
'long_time', 'med_mem'
),
"count_rs_ids_in_release_files": (
["count-ids-script-path", "taxonomy-id", "assembly-accession", "species-release-folder"],
'long_time', 'med_mem'
),
"validate_rs_release_files": (
["private-config-xml-file", "profile", "taxonomy-id", "assembly-accession",
"release-species-inventory-table", "release-version", "species-release-folder"],
'long_time', 'med_mem'
),
'update_sequence_names_to_ena': (
["taxonomy-id", "assembly-accession", "species-release-folder",'sequence-name-converter-path',
'bcftools-path'],
'long_time', 'med_mem'
),
"update_release_status_for_assembly": (
["private-config-xml-file", "profile", "release-species-inventory-table", "taxonomy-id",
"assembly-accession", "release-version"],
'short_time', 'small_mem'
)
}
)

workflow_process_template_for_nextflow = """
process {workflow-process-name} {{

{cluster-options}
label '{label-time}', '{label-memory}'

input:
val flag from {previous-process-output-flag}
output:
Expand Down Expand Up @@ -99,7 +129,7 @@ def get_release_properties_for_current_assembly(species_release_properties, asse


def get_nextflow_process_definition(assembly_release_properties, workflow_process_name, workflow_process_args,
process_name_suffix=None):
process_name_suffix=None, label_time=None, label_memory=None):
if process_name_suffix is None:
process_name_suffix = assembly_release_properties["assembly-accession"].replace('.', '_')
release_properties = copy.deepcopy(assembly_release_properties)
Expand All @@ -109,8 +139,8 @@ def get_nextflow_process_definition(assembly_release_properties, workflow_proces
" ".join(["--{0} {1}"
.format(arg, release_properties[arg])
for arg in workflow_process_args]))
release_properties["cluster-options"] = 'clusterOptions "-g /accession"' if "run_release_for_assembly" \
in workflow_process_name else ""
release_properties['label-time'] = label_time or 'long_time'
release_properties['label-memory'] = label_memory or 'med_mem'
return workflow_process_template_for_nextflow.format(**release_properties)


Expand All @@ -137,13 +167,25 @@ def prepare_release_workflow_file_for_species(common_release_properties, taxonom
header = "#!/usr/bin/env nextflow"
workflow_file_handle.write(header + "\n")

for workflow_process_name, workflow_process_args in workflow_process_arguments_map.items():
for workflow_process_name, workflow_process_info in workflow_process_arguments_map.items():
workflow_process_args = workflow_process_info[0]
label_time = label_memory = None
if len(workflow_process_info) > 1:
label_time = workflow_process_info[1]
if len(workflow_process_info) > 2:
label_memory = workflow_process_info[2]
release_assembly_properties["current-process-output-flag"] = "flag" + str(process_index)

workflow_file_handle.write(
get_nextflow_process_definition(release_assembly_properties, workflow_process_name,
workflow_process_args,
process_name_suffix=assembly_accession.replace(".", "_")))
get_nextflow_process_definition(
release_assembly_properties,
workflow_process_name,
workflow_process_args,
process_name_suffix=assembly_accession.replace(".", "_"),
label_time=label_time,
label_memory=label_memory
)
)
workflow_file_handle.write("\n")
process_index += 1
# Set the flag that will capture the output status of the current process
Expand Down
Loading